{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "BcLSRzzIgepr"
   },
   "source": [
    "# Setup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 173,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "Y0-vqB5tXGNi",
    "outputId": "d6e62d31-d110-4e61-b48e-2e87b8088bc3"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[1;33mW: \u001b[0mGPG error: http://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64  InRelease: The following signatures couldn't be verified because the public key is not available: NO_PUBKEY A4B469963BF863CC\u001b[0m\r\n",
      "\u001b[1;31mE: \u001b[0mThe repository 'http://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64  InRelease' is not signed.\u001b[0m\r\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The following packages were automatically installed and are no longer required:\n",
      "  libaccinj64-11.5 libasyncns0 libbabeltrace1 libboost-regex1.74.0 libcub-dev\n",
      "  libcublas11 libcublaslt11 libcudart11.0 libcufft10 libcufftw10 libcupti-dev\n",
      "  libcupti-doc libcupti11.5 libcurand10 libcusolver11 libcusolvermg11\n",
      "  libcusparse11 libdebuginfod-common libdebuginfod1 libdouble-conversion3\n",
      "  libegl-dev libflac8 libgl-dev libgl1-mesa-dev libgles-dev libgles1 libgles2\n",
      "  libglvnd-core-dev libglvnd-dev libglx-dev libipt2 libnppc11 libnppial11\n",
      "  libnppicc11 libnppidei11 libnppif11 libnppig11 libnppim11 libnppist11\n",
      "  libnppisu11 libnppitc11 libnpps11 libnvblas11 libnvjpeg11\n",
      "  libnvrtc-builtins11.5 libnvrtc11.2 libnvtoolsext1 libnvvm4 libogg0\n",
      "  libopengl-dev libopengl0 libopus0 libpcre2-16-0 libpulse0 libqt5core5a\n",
      "  libqt5dbus5 libqt5network5 libsndfile1 libsource-highlight-common\n",
      "  libsource-highlight4v5 libtbb-dev libtbb12 libtbbmalloc2 libthrust-dev\n",
      "  libvdpau-dev libvorbis0a libvorbisenc2 libxcb-icccm4 libxcb-image0\n",
      "  libxcb-keysyms1 libxcb-render-util0 libxcb-util1 libxcb-xinerama0\n",
      "  libxcb-xkb1 libxkbcommon-x11-0 node-html5shiv nsight-compute\n",
      "  nsight-compute-target nvidia-cuda-gdb nvidia-cuda-toolkit-doc\n",
      "  nvidia-opencl-dev ocl-icd-libopencl1 ocl-icd-opencl-dev opencl-c-headers\n",
      "  opencl-clhpp-headers openjdk-8-jre qttranslations5-l10n\n",
      "Use 'sudo apt autoremove' to remove them.\n",
      "Get more security updates through Ubuntu Pro with 'esm-apps' enabled:\n",
      "  python3-jupyter-core maven jupyter libjs-jquery-ui libmaven3-core-java\n",
      "  python3-scipy jupyter-core\n",
      "Learn more about Ubuntu Pro at https://ubuntu.com/pro\n",
      "The following packages have been kept back:\n",
      "  python3-update-manager update-manager-core\n",
      "0 upgraded, 0 newly installed, 0 to remove and 2 not upgraded.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "PPA publishes dbgsym, you may need to include 'main/debug' component\n",
      "Repository: 'deb https://ppa.launchpadcontent.net/graphics-drivers/ppa/ubuntu/ jammy main'\n",
      "Description:\n",
      "Fresh drivers from upstream, currently shipping Nvidia.\n",
      "\n",
      "## Current releases\n",
      "\n",
      "Current production branch release: 535.171.04\n",
      "Current new feature branch release: 550.78\n",
      "Current beta release: 550.40.07\n",
      "\n",
      "## Legacy releases\n",
      "\n",
      "470.239.06 (x86_64) - GKxxx “Kepler” GPUs\n",
      "390.157 (x86 / x86_64 / ARM) - GF1xx “Fermi” GPUs (*​)\n",
      "340.108 (x86 / x86_64) - GeForce 8 and 9 series GPUs (*​)\n",
      "304.137 (x86 / x86_64) - GeForce 6 and 7 series GPUs (*​)\n",
      "173.14.39 (x86 / x86_64) - GeForce 5 series GPUs (*​)\n",
      "96.43.23 (x86 / x86_64) - GeForce 2 through GeForce 4 series GPUs (*​)\n",
      "71.86.15 (x86 / x86_64) - Riva TNT, TNT2, GeForce, and some GeForce 2 GPUs (*​)\n",
      "(*​) These releases are no longer being maintained. Please see Support timeframes for Unix legacy GPU releases for more details. https://nvidia.custhelp.com/app/answers/detail/a_id/3142\n",
      "\n",
      "## What we're working on right now:\n",
      "\n",
      "- Normal driver updates\n",
      "- Help Wanted: Mesa Updates for Intel/AMD users, ping us if you want to help do this work, we're shorthanded.\n",
      "\n",
      "## WARNINGS:\n",
      "\n",
      "This PPA is currently in testing, you should be experienced with packaging before you dive in here:\n",
      "\n",
      "Volunteers welcome!\n",
      "\n",
      "### How you can help:\n",
      "\n",
      "## Install PTS and benchmark your gear:\n",
      "\n",
      "    sudo apt-get install phoronix-test-suite\n",
      "\n",
      "Run the benchmark:\n",
      "\n",
      "    phoronix-test-suite default-benchmark openarena xonotic tesseract gputest unigine-valley\n",
      "\n",
      "and then say yes when it asks you to submit your results to openbechmarking.org. Then grab a cup of coffee, it takes a bit for the benchmarks to run. Depending on the version of Ubuntu you're using it might preferable for you to grabs PTS from upstream directly: http://www.phoronix-test-suite.com/?k=downloads\n",
      "\n",
      "## Share your results with the community:\n",
      "\n",
      "Post a link to your results (or any other feedback to): https://launchpad.net/~graphics-drivers-testers\n",
      "\n",
      "Remember to rerun and resubmit the benchmarks after driver upgrades, this will allow us to gather a bunch of data on performance that we can share with everybody.\n",
      "\n",
      "If you run into old documentation referring to other PPAs, you can help us by consolidating references to this PPA.\n",
      "\n",
      "If someone wants to go ahead and start prototyping on `software-properties-gtk` on what the GUI should look like, please start hacking!\n",
      "\n",
      "## Help us Help You!\n",
      "\n",
      "We use the donation funds to get the developers hardware to test and upload these drivers, please consider donating to the \"community\" slider on the donation page if you're loving this PPA:\n",
      "\n",
      "http://www.ubuntu.com/download/desktop/contribute\n",
      "More info: https://launchpad.net/~graphics-drivers/+archive/ubuntu/ppa\n",
      "Adding repository.\n",
      "Found existing deb entry in /etc/apt/sources.list.d/graphics-drivers-ubuntu-ppa-jammy.list\n",
      "Adding deb entry to /etc/apt/sources.list.d/graphics-drivers-ubuntu-ppa-jammy.list\n",
      "Found existing deb-src entry in /etc/apt/sources.list.d/graphics-drivers-ubuntu-ppa-jammy.list\n",
      "Adding disabled deb-src entry to /etc/apt/sources.list.d/graphics-drivers-ubuntu-ppa-jammy.list\n",
      "Adding key to /etc/apt/trusted.gpg.d/graphics-drivers-ubuntu-ppa.gpg with fingerprint 2388FF3BE10A76F638F80723FCAE110B1118213C\n",
      "Get:1 http://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64  InRelease [1581 B]\n",
      "Err:1 http://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64  InRelease\n",
      "  The following signatures couldn't be verified because the public key is not available: NO_PUBKEY A4B469963BF863CC\n",
      "Hit:2 http://archive.ubuntu.com/ubuntu jammy InRelease                         \n",
      "Hit:3 http://security.ubuntu.com/ubuntu jammy-security InRelease               \n",
      "Hit:4 http://archive.ubuntu.com/ubuntu jammy-updates InRelease                 \n",
      "Hit:5 http://archive.ubuntu.com/ubuntu jammy-backports InRelease\n",
      "Hit:6 https://ppa.launchpadcontent.net/graphics-drivers/ppa/ubuntu jammy InRelease\n",
      "Reading package lists... Done\n",
      "W: GPG error: http://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64  InRelease: The following signatures couldn't be verified because the public key is not available: NO_PUBKEY A4B469963BF863CC\n",
      "E: The repository 'http://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64  InRelease' is not signed.\n",
      "N: Updating from such a repository can't be done securely, and is therefore disabled by default.\n",
      "N: See apt-secure(8) manpage for repository creation and user configuration details.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[1;33mW: \u001b[0mGPG error: http://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64  InRelease: The following signatures couldn't be verified because the public key is not available: NO_PUBKEY A4B469963BF863CC\u001b[0m\n",
      "\u001b[1;31mE: \u001b[0mThe repository 'http://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64  InRelease' is not signed.\u001b[0m\n",
      "/bin/bash: line 1: ubuntu-drivers: command not found\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
     ]
    }
   ],
   "source": [
    "!sudo apt update -qq \n",
    "!sudo apt upgrade -y -qq\n",
    "!sudo add-apt-repository -y ppa:graphics-drivers/ppa\n",
    "!sudo apt update -qq\n",
    "!ubuntu-drivers devices"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 174,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: torch in /usr/local/lib/python3.10/dist-packages (2.4.0)\n",
      "Requirement already satisfied: nvidia-cuda-runtime-cu12==12.1.105 in /usr/local/lib/python3.10/dist-packages (from torch) (12.1.105)\n",
      "Requirement already satisfied: nvidia-cusparse-cu12==12.1.0.106 in /usr/local/lib/python3.10/dist-packages (from torch) (12.1.0.106)\n",
      "Requirement already satisfied: nvidia-nccl-cu12==2.20.5 in /usr/local/lib/python3.10/dist-packages (from torch) (2.20.5)\n",
      "Requirement already satisfied: triton==3.0.0 in /usr/local/lib/python3.10/dist-packages (from torch) (3.0.0)\n",
      "Requirement already satisfied: nvidia-cusolver-cu12==11.4.5.107 in /usr/local/lib/python3.10/dist-packages (from torch) (11.4.5.107)\n",
      "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from torch) (3.15.4)\n",
      "Requirement already satisfied: typing-extensions>=4.8.0 in /usr/local/lib/python3.10/dist-packages (from torch) (4.12.2)\n",
      "Requirement already satisfied: fsspec in /usr/local/lib/python3.10/dist-packages (from torch) (2024.3.1)\n",
      "Requirement already satisfied: nvidia-cufft-cu12==11.0.2.54 in /usr/local/lib/python3.10/dist-packages (from torch) (11.0.2.54)\n",
      "Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch) (3.3)\n",
      "Requirement already satisfied: sympy in /usr/lib/python3/dist-packages (from torch) (1.9)\n",
      "Requirement already satisfied: nvidia-curand-cu12==10.3.2.106 in /usr/local/lib/python3.10/dist-packages (from torch) (10.3.2.106)\n",
      "Requirement already satisfied: nvidia-cuda-nvrtc-cu12==12.1.105 in /usr/local/lib/python3.10/dist-packages (from torch) (12.1.105)\n",
      "Requirement already satisfied: jinja2 in /usr/lib/python3/dist-packages (from torch) (3.0.3)\n",
      "Requirement already satisfied: nvidia-cuda-cupti-cu12==12.1.105 in /usr/local/lib/python3.10/dist-packages (from torch) (12.1.105)\n",
      "Requirement already satisfied: nvidia-cudnn-cu12==9.1.0.70 in /usr/local/lib/python3.10/dist-packages (from torch) (9.1.0.70)\n",
      "Requirement already satisfied: nvidia-nvtx-cu12==12.1.105 in /usr/local/lib/python3.10/dist-packages (from torch) (12.1.105)\n",
      "Requirement already satisfied: nvidia-cublas-cu12==12.1.3.1 in /usr/local/lib/python3.10/dist-packages (from torch) (12.1.3.1)\n",
      "Requirement already satisfied: nvidia-nvjitlink-cu12 in /usr/local/lib/python3.10/dist-packages (from nvidia-cusolver-cu12==11.4.5.107->torch) (12.5.82)\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
      "\u001b[0m"
     ]
    }
   ],
   "source": [
    "!pip install torch --quiet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 175,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 175,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import torch\n",
    "\n",
    "torch.cuda.is_available()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "_PfU4knm7kpa"
   },
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 176,
   "metadata": {
    "id": "r2PmDR6PArZk"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
     ]
    }
   ],
   "source": [
    "file_content=\"\"\"\n",
    "keyboard-configuration  keyboard-configuration/model  select  Generic 105-key (Intl) PC\n",
    "keyboard-configuration  keyboard-configuration/layout select  English (US)\n",
    "keyboard-configuration  keyboard-configuration/variant select  English (US)\n",
    "keyboard-configuration  keyboard-configuration/optionscode string\n",
    "\"\"\"\n",
    "\n",
    "conffile = open(\"/etc/default/keyboard-configuration\", \"w\")\n",
    "conffile.write(file_content)\n",
    "conffile.close()\n",
    "\n",
    "!sudo debconf-set-selections < /etc/default/keyboard-configuration\n",
    "\n",
    "!sudo DEBIAN_FRONTEND=noninteractive dpkg --configure -a\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 177,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "Qzc5qY1cDtlS",
    "outputId": "852f6a5b-f806-44e0-cd1d-5f568c95a5a5"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[1;31mE: \u001b[0mUnable to locate package cuda\u001b[0m\r\n"
     ]
    }
   ],
   "source": [
    "!sudo apt install cuda -qq\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 178,
   "metadata": {
    "id": "VUvaIjE3XpIJ"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "nvidia-driver-470 is already the newest version (470.256.02-0ubuntu0.22.04.1).\r\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The following packages were automatically installed and are no longer required:\r\n",
      "  libaccinj64-11.5 libasyncns0 libbabeltrace1 libboost-regex1.74.0 libcub-dev\r\n",
      "  libcublas11 libcublaslt11 libcudart11.0 libcufft10 libcufftw10 libcupti-dev\r\n",
      "  libcupti-doc libcupti11.5 libcurand10 libcusolver11 libcusolvermg11\r\n",
      "  libcusparse11 libdebuginfod-common libdebuginfod1 libdouble-conversion3\r\n",
      "  libegl-dev libflac8 libgl-dev libgl1-mesa-dev libgles-dev libgles1 libgles2\r\n",
      "  libglvnd-core-dev libglvnd-dev libglx-dev libipt2 libnppc11 libnppial11\r\n",
      "  libnppicc11 libnppidei11 libnppif11 libnppig11 libnppim11 libnppist11\r\n",
      "  libnppisu11 libnppitc11 libnpps11 libnvblas11 libnvjpeg11\r\n",
      "  libnvrtc-builtins11.5 libnvrtc11.2 libnvtoolsext1 libnvvm4 libogg0\r\n",
      "  libopengl-dev libopengl0 libopus0 libpcre2-16-0 libpulse0 libqt5core5a\r\n",
      "  libqt5dbus5 libqt5network5 libsndfile1 libsource-highlight-common\r\n",
      "  libsource-highlight4v5 libtbb-dev libtbb12 libtbbmalloc2 libthrust-dev\r\n",
      "  libvdpau-dev libvorbis0a libvorbisenc2 libxcb-icccm4 libxcb-image0\r\n",
      "  libxcb-keysyms1 libxcb-render-util0 libxcb-util1 libxcb-xinerama0\r\n",
      "  libxcb-xkb1 libxkbcommon-x11-0 node-html5shiv nsight-compute\r\n",
      "  nsight-compute-target nvidia-cuda-gdb nvidia-cuda-toolkit-doc\r\n",
      "  nvidia-opencl-dev ocl-icd-libopencl1 ocl-icd-opencl-dev opencl-c-headers\r\n",
      "  opencl-clhpp-headers openjdk-8-jre qttranslations5-l10n\r\n",
      "Use 'sudo apt autoremove' to remove them.\r\n",
      "0 upgraded, 0 newly installed, 0 to remove and 2 not upgraded.\r\n"
     ]
    }
   ],
   "source": [
    "!sudo apt install -y -qq  nvidia-driver-470\n",
    "#restart runtime now"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 179,
   "metadata": {
    "id": "W90ZQ3MXDAji"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "modprobe: FATAL: Module nvidia not found in directory /lib/modules/5.15.133.1-microsoft-standard-WSL2\r\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
     ]
    }
   ],
   "source": [
    "!sudo modprobe nvidia\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 180,
   "metadata": {
    "id": "cJpYLE_QHMfd"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/bin/bash: line 1: nvcc: command not found\r\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
     ]
    }
   ],
   "source": [
    "!nvcc --version"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 181,
   "metadata": {
    "id": "kk11MshyKshD"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/bin/bash: line 1: nvcc: command not found\r\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
     ]
    }
   ],
   "source": [
    "!nvcc  --list-gpu-arch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 182,
   "metadata": {
    "id": "0PnfEWweLq-b"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--2024-07-28 17:18:13--  https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/cuda-ubuntu2004.pin\n",
      "Resolving developer.download.nvidia.com (developer.download.nvidia.com)... 152.195.19.142\n",
      "Connecting to developer.download.nvidia.com (developer.download.nvidia.com)|152.195.19.142|:443... connected.\n",
      "HTTP request sent, awaiting response... 200 OK\n",
      "Length: 190 [application/octet-stream]\n",
      "Saving to: ‘cuda-ubuntu2004.pin’\n",
      "\n",
      "cuda-ubuntu2004.pin 100%[===================>]     190  --.-KB/s    in 0s      \n",
      "\n",
      "2024-07-28 17:18:13 (21.6 MB/s) - ‘cuda-ubuntu2004.pin’ saved [190/190]\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Warning: apt-key is deprecated. Manage keyring files in trusted.gpg.d instead (see apt-key(8)).\n",
      "Executing: /tmp/apt-key-gpghome.0S5ysqf8XO/gpg.1.sh --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/7fa2af80.pub\n",
      "gpg: requesting key from 'https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/7fa2af80.pub'\n",
      "gpg: key F60F4B3D7FA2AF80: \"cudatools <cudatools@nvidia.com>\" not changed\n",
      "gpg: Total number processed: 1\n",
      "gpg:              unchanged: 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Repository: 'deb http://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/ /'\n",
      "Description:\n",
      "Archive for codename: / components: \n",
      "More info: http://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/\n",
      "Adding repository.\n",
      "Found existing deb entry in /etc/apt/sources.list.d/archive_uri-http_developer_download_nvidia_com_compute_cuda_repos_ubuntu2004_x86_64_-jammy.list\n",
      "Adding deb entry to /etc/apt/sources.list.d/archive_uri-http_developer_download_nvidia_com_compute_cuda_repos_ubuntu2004_x86_64_-jammy.list\n",
      "Found existing deb-src entry in /etc/apt/sources.list.d/archive_uri-http_developer_download_nvidia_com_compute_cuda_repos_ubuntu2004_x86_64_-jammy.list\n",
      "Adding disabled deb-src entry to /etc/apt/sources.list.d/archive_uri-http_developer_download_nvidia_com_compute_cuda_repos_ubuntu2004_x86_64_-jammy.list\n",
      "Get:1 http://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64  InRelease [1581 B]\n",
      "Err:1 http://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64  InRelease\n",
      "  The following signatures couldn't be verified because the public key is not available: NO_PUBKEY A4B469963BF863CC\n",
      "Hit:2 http://archive.ubuntu.com/ubuntu jammy InRelease                         \n",
      "Hit:3 http://archive.ubuntu.com/ubuntu jammy-updates InRelease                 \n",
      "Hit:4 http://security.ubuntu.com/ubuntu jammy-security InRelease               \n",
      "Hit:5 http://archive.ubuntu.com/ubuntu jammy-backports InRelease               \n",
      "Hit:6 https://ppa.launchpadcontent.net/graphics-drivers/ppa/ubuntu jammy InRelease\n",
      "Reading package lists... Done\n",
      "W: GPG error: http://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64  InRelease: The following signatures couldn't be verified because the public key is not available: NO_PUBKEY A4B469963BF863CC\n",
      "E: The repository 'http://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64  InRelease' is not signed.\n",
      "N: Updating from such a repository can't be done securely, and is therefore disabled by default.\n",
      "N: See apt-secure(8) manpage for repository creation and user configuration details.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "W: GPG error: http://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64  InRelease: The following signatures couldn't be verified because the public key is not available: NO_PUBKEY A4B469963BF863CC\n",
      "E: The repository 'http://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64  InRelease' is not signed.\n",
      "E: Unable to locate package cuda\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
     ]
    }
   ],
   "source": [
    "!wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/cuda-ubuntu2004.pin\n",
    "!sudo mv cuda-ubuntu2004.pin /etc/apt/preferences.d/cuda-repository-pin-600\n",
    "!sudo apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/7fa2af80.pub\n",
    "!sudo add-apt-repository -y \"deb http://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/ /\"\n",
    "!sudo apt-get update -qq\n",
    "!sudo apt-get -y install cuda -qq\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 183,
   "metadata": {
    "id": "uVMFhhTGIMKx"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
     ]
    }
   ],
   "source": [
    "!sudo apt-get install pciutils -qq"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 184,
   "metadata": {
    "id": "fEEYac3QIIFB"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
     ]
    }
   ],
   "source": [
    "!lspci | grep -i vga"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 185,
   "metadata": {
    "id": "HBon37tnHpdV"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'NVIDIA GeForce RTX 4090 Laptop GPU'"
      ]
     },
     "execution_count": 185,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import torch\n",
    "torch.cuda.is_available()\n",
    "torch.cuda.get_device_name(0)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 186,
   "metadata": {
    "id": "wKPiSbpiXoMD"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Sun Jul 28 17:18:27 2024       \r\n",
      "+-----------------------------------------------------------------------------+\r\n",
      "| NVIDIA-SMI 470.256.02   Driver Version: 546.32       CUDA Version: 12.3     |\r\n",
      "|-------------------------------+----------------------+----------------------+\r\n",
      "| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |\r\n",
      "| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |\r\n",
      "|                               |                      |               MIG M. |\r\n",
      "|===============================+======================+======================|\r\n",
      "|   0  NVIDIA GeForce ...  On   | 00000000:01:00.0 Off |                  N/A |\r\n",
      "| N/A   37C    P8     4W / 175W |   3731MiB / 16376MiB |      0%      Default |\r\n",
      "|                               |                      |                  N/A |\r\n",
      "+-------------------------------+----------------------+----------------------+\r\n",
      "                                                                               \r\n",
      "+-----------------------------------------------------------------------------+\r\n",
      "| Processes:                                                                  |\r\n",
      "|  GPU   GI   CI        PID   Type   Process name                  GPU Memory |\r\n",
      "|        ID   ID                                                   Usage      |\r\n",
      "|=============================================================================|\r\n",
      "|    0   N/A  N/A    224772      C   /python3.10                     N/A      |\r\n",
      "+-----------------------------------------------------------------------------+\r\n"
     ]
    }
   ],
   "source": [
    "!nvidia-smi"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "o3mYU5CfSB-Y"
   },
   "source": [
    "# Section 3.1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 187,
   "metadata": {
    "id": "Rez0J37bUt32"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\r\n",
      "\u001b[0m"
     ]
    }
   ],
   "source": [
    "!pip install transformers  google --quiet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 188,
   "metadata": {
    "id": "-ffWKZKEUhAk"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of BertForSequenceClassification were not initialized from the model checkpoint at bert-base-uncased and are newly initialized: ['classifier.bias', 'classifier.weight']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    }
   ],
   "source": [
    "from transformers import BertTokenizer, BertForSequenceClassification\n",
    "\n",
    "model_name = 'bert-base-uncased'\n",
    "tokenizer = BertTokenizer.from_pretrained(model_name)\n",
    "model = BertForSequenceClassification.from_pretrained(model_name)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 189,
   "metadata": {
    "id": "6mjWGLVXZy9t"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\r\n",
      "\u001b[0m"
     ]
    }
   ],
   "source": [
    "!pip install openai google SentencePiece  --q"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 190,
   "metadata": {
    "id": "TrmpbE1zTMeL"
   },
   "outputs": [
    {
     "ename": "AuthenticationError",
     "evalue": "Error code: 401 - {'error': {'message': 'Incorrect API key provided: <YOUR KE***ERE>. You can find your API key at https://platform.openai.com/account/api-keys.', 'type': 'invalid_request_error', 'param': None, 'code': 'invalid_api_key'}}",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mAuthenticationError\u001b[0m                       Traceback (most recent call last)",
      "\u001b[0;32m/tmp/ipykernel_224772/3233283554.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      7\u001b[0m \u001b[0mprompt\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"Translate the following English text to French: 'Hello, how are you?'\"\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      8\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 9\u001b[0;31m response = client.chat.completions.create(\n\u001b[0m\u001b[1;32m     10\u001b[0m         \u001b[0mmodel\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"gpt-4o\"\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     11\u001b[0m         messages=[\n",
      "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/openai/_utils/_utils.py\u001b[0m in \u001b[0;36mwrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m    275\u001b[0m                         \u001b[0mmsg\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34mf\"Missing required argument: {quote(missing[0])}\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    276\u001b[0m                 \u001b[0;32mraise\u001b[0m \u001b[0mTypeError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmsg\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 277\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    278\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    279\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mwrapper\u001b[0m  \u001b[0;31m# type: ignore\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/openai/resources/chat/completions.py\u001b[0m in \u001b[0;36mcreate\u001b[0;34m(self, messages, model, frequency_penalty, function_call, functions, logit_bias, logprobs, max_tokens, n, parallel_tool_calls, presence_penalty, response_format, seed, service_tier, stop, stream, stream_options, temperature, tool_choice, tools, top_logprobs, top_p, user, extra_headers, extra_query, extra_body, timeout)\u001b[0m\n\u001b[1;32m    644\u001b[0m         \u001b[0mtimeout\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mfloat\u001b[0m \u001b[0;34m|\u001b[0m \u001b[0mhttpx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTimeout\u001b[0m \u001b[0;34m|\u001b[0m \u001b[0;32mNone\u001b[0m \u001b[0;34m|\u001b[0m \u001b[0mNotGiven\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mNOT_GIVEN\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    645\u001b[0m     ) -> ChatCompletion | Stream[ChatCompletionChunk]:\n\u001b[0;32m--> 646\u001b[0;31m         return self._post(\n\u001b[0m\u001b[1;32m    647\u001b[0m             \u001b[0;34m\"/chat/completions\"\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    648\u001b[0m             body=maybe_transform(\n",
      "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/openai/_base_client.py\u001b[0m in \u001b[0;36mpost\u001b[0;34m(self, path, cast_to, body, options, files, stream, stream_cls)\u001b[0m\n\u001b[1;32m   1264\u001b[0m             \u001b[0mmethod\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"post\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0murl\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mjson_data\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mbody\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfiles\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mto_httpx_files\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfiles\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0moptions\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1265\u001b[0m         )\n\u001b[0;32m-> 1266\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mcast\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mResponseT\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrequest\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcast_to\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mopts\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstream\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mstream\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstream_cls\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mstream_cls\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1267\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1268\u001b[0m     def patch(\n",
      "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/openai/_base_client.py\u001b[0m in \u001b[0;36mrequest\u001b[0;34m(self, cast_to, options, remaining_retries, stream, stream_cls)\u001b[0m\n\u001b[1;32m    940\u001b[0m         \u001b[0mstream_cls\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mtype\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0m_StreamT\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m|\u001b[0m \u001b[0;32mNone\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    941\u001b[0m     ) -> ResponseT | _StreamT:\n\u001b[0;32m--> 942\u001b[0;31m         return self._request(\n\u001b[0m\u001b[1;32m    943\u001b[0m             \u001b[0mcast_to\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcast_to\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    944\u001b[0m             \u001b[0moptions\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0moptions\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/openai/_base_client.py\u001b[0m in \u001b[0;36m_request\u001b[0;34m(self, cast_to, options, remaining_retries, stream, stream_cls)\u001b[0m\n\u001b[1;32m   1044\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1045\u001b[0m             \u001b[0mlog\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdebug\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Re-raising status error\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1046\u001b[0;31m             \u001b[0;32mraise\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_make_status_error_from_response\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0merr\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mresponse\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1047\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1048\u001b[0m         return self._process_response(\n",
      "\u001b[0;31mAuthenticationError\u001b[0m: Error code: 401 - {'error': {'message': 'Incorrect API key provided: <YOUR KE***ERE>. You can find your API key at https://platform.openai.com/account/api-keys.', 'type': 'invalid_request_error', 'param': None, 'code': 'invalid_api_key'}}"
     ]
    }
   ],
   "source": [
    "from openai import OpenAI\n",
    "#from google.colab import userdata\n",
    "\n",
    "#client = OpenAI(api_key=userdata.get('open_ai_key'))\n",
    "client = OpenAI(api_key=\"<YOUR KEY HERE>\")\n",
    "\n",
    "prompt=\"Translate the following English text to French: 'Hello, how are you?'\",\n",
    "\n",
    "response = client.chat.completions.create(\n",
    "        model=\"gpt-4o\",\n",
    "        messages=[\n",
    "            {\"role\": \"system\", \"content\": \"You are a helpful assistant that translates text.\"},\n",
    "            {\"role\": \"user\", \"content\": str(prompt) }\n",
    "        ],\n",
    "        max_tokens=150,\n",
    "        temperature=0.7,\n",
    "    )\n",
    "\n",
    "print(response.choices[0].message.content.strip())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 191,
   "metadata": {
    "id": "dI2pFRzPTMyk"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
     ]
    }
   ],
   "source": [
    "from transformers import T5Tokenizer, T5ForConditionalGeneration\n",
    "\n",
    "model_name = 't5-base'\n",
    "tokenizer = T5Tokenizer.from_pretrained(model_name)\n",
    "model = T5ForConditionalGeneration.from_pretrained(model_name)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 192,
   "metadata": {
    "id": "9XPgyi53TbLN"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.dense.bias', 'classifier.dense.weight', 'classifier.out_proj.bias', 'classifier.out_proj.weight']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    }
   ],
   "source": [
    "from transformers import RobertaTokenizer, RobertaForSequenceClassification\n",
    "\n",
    "model_name = 'roberta-base'\n",
    "tokenizer = RobertaTokenizer.from_pretrained(model_name)\n",
    "model = RobertaForSequenceClassification.from_pretrained(model_name)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "hTvNpAq-hIhP"
   },
   "source": [
    "# Section 3.2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 193,
   "metadata": {
    "id": "6TMIMchsabIY",
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
      "\u001b[0mNote: you may need to restart the kernel to use updated packages.\n"
     ]
    }
   ],
   "source": [
    "pip install torch torchvision datasets -q"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 194,
   "metadata": {
    "id": "jZGjn7iqrHLP"
   },
   "outputs": [],
   "source": [
    "import torch.nn as nn\n",
    "\n",
    "# Define a simple neural network model\n",
    "class SimpleCNN(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(SimpleCNN, self).__init__()\n",
    "        self.conv1 = nn.Conv2d(1, 32, kernel_size=3, stride=1, padding=1)\n",
    "        self.conv2 = nn.Conv2d(32, 64, kernel_size=3, stride=1, padding=1)\n",
    "        self.fc1 = nn.Linear(64*7*7, 128)\n",
    "        self.fc2 = nn.Linear(128, 10)\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = torch.relu(self.conv1(x))\n",
    "        x = torch.max_pool2d(x, 2)\n",
    "        x = torch.relu(self.conv2(x))\n",
    "        x = torch.max_pool2d(x, 2)\n",
    "        x = x.view(-1, 64*7*7)\n",
    "        x = torch.relu(self.fc1(x))\n",
    "        x = self.fc2(x)\n",
    "        return x\n",
    "\n",
    "def create_model(learning_rate, batch_size=5):\n",
    "    model = SimpleCNN().cuda()\n",
    "    return model\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 195,
   "metadata": {
    "id": "leV2w-qdaYuY"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [1], Loss: 0.1217, Accuracy: 96.27%\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "from torch.utils.data import DataLoader\n",
    "from torchvision import datasets, transforms\n",
    "from torch.amp import GradScaler, autocast\n",
    "\n",
    "# Load MNIST dataset\n",
    "transform = transforms.Compose([transforms.ToTensor(), transforms.Normalize((0.5,), (0.5,))])\n",
    "train_dataset = datasets.MNIST(root='./data', train=True, download=True, transform=transform)\n",
    "train_loader = DataLoader(train_dataset, batch_size=32, shuffle=True)\n",
    "\n",
    "# Initialize model, loss function, and optimizer\n",
    "model = create_model(learning_rate=2e-5)\n",
    "criterion = nn.CrossEntropyLoss()\n",
    "optimizer = optim.Adam(model.parameters(), lr=0.001)\n",
    "\n",
    "# Initialize GradScaler for mixed-precision training\n",
    "scaler = GradScaler('cuda')\n",
    "\n",
    "# Training loop\n",
    "for epoch in range(1):  # Training for 1 epoch for demonstration\n",
    "    correct = 0\n",
    "    total = 0\n",
    "    running_loss = 0.0\n",
    "\n",
    "    for data, target in train_loader:\n",
    "        data, target = data.cuda(), target.cuda()  # Move data to GPU\n",
    "\n",
    "        optimizer.zero_grad()\n",
    "\n",
    "        # Forward pass with autocast for mixed-precision\n",
    "        with autocast('cuda'):\n",
    "            output = model(data)\n",
    "            loss = criterion(output, target)\n",
    "\n",
    "        # Backward pass and optimization\n",
    "        scaler.scale(loss).backward()\n",
    "        scaler.step(optimizer)\n",
    "        scaler.update()\n",
    "\n",
    "        # Calculate running loss\n",
    "        running_loss += loss.item()\n",
    "\n",
    "        # Calculate accuracy\n",
    "        _, predicted = torch.max(output.data, 1)\n",
    "        total += target.size(0)\n",
    "        correct += (predicted == target).sum().item()\n",
    "\n",
    "    epoch_loss = running_loss / len(train_loader)\n",
    "    epoch_accuracy = 100 * correct / total\n",
    "\n",
    "    print(f'Epoch [{epoch+1}], Loss: {epoch_loss:.4f}, Accuracy: {epoch_accuracy:.2f}%')\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 196,
   "metadata": {
    "id": "lYYFsrmiVy6Y"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\r\n",
      "tensorflow 2.17.0 requires numpy<2.0.0,>=1.23.5; python_version <= \"3.11\", but you have numpy 1.22.4 which is incompatible.\u001b[0m\u001b[31m\r\n",
      "\u001b[0m\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\r\n",
      "\u001b[0m"
     ]
    }
   ],
   "source": [
    "!pip install optuna numpy==1.22.4 -q"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 197,
   "metadata": {
    "id": "dcqdoltdUGgM"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of BertForSequenceClassification were not initialized from the model checkpoint at bert-base-uncased and are newly initialized: ['classifier.bias', 'classifier.weight']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
      "[I 2024-07-28 17:22:53,091] A new study created in memory with name: no-name-047bcaa8-b94e-473d-b239-a17fee356b08\n",
      "/usr/local/lib/python3.10/dist-packages/accelerate/accelerator.py:488: FutureWarning: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead.\n",
      "  self.scaler = torch.cuda.amp.GradScaler(**kwargs)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='345' max='345' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [345/345 00:22, Epoch 3/3]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Epoch</th>\n",
       "      <th>Training Loss</th>\n",
       "      <th>Validation Loss</th>\n",
       "      <th>Accuracy</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.471443</td>\n",
       "      <td>0.764706</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.379084</td>\n",
       "      <td>0.845588</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.413528</td>\n",
       "      <td>0.862745</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='13' max='13' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [13/13 00:00]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-07-28 17:23:23,720] Trial 0 finished with value: 0.8627450980392157 and parameters: {'lr': 1.1338636714975849e-05, 'learning_rate': 0.00011661427090367127, 'batch_size': 32, 'weight_decay': 9.066757241590247e-05}. Best is trial 0 with value: 0.8627450980392157.\n",
      "/usr/local/lib/python3.10/dist-packages/accelerate/accelerator.py:488: FutureWarning: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead.\n",
      "  self.scaler = torch.cuda.amp.GradScaler(**kwargs)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='690' max='690' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [690/690 00:48, Epoch 3/3]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Epoch</th>\n",
       "      <th>Training Loss</th>\n",
       "      <th>Validation Loss</th>\n",
       "      <th>Accuracy</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.602651</td>\n",
       "      <td>0.718137</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.874194</td>\n",
       "      <td>0.745098</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>0.438800</td>\n",
       "      <td>0.611717</td>\n",
       "      <td>0.794118</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='26' max='26' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [26/26 00:00]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-07-28 17:24:18,815] Trial 1 finished with value: 0.7941176470588235 and parameters: {'lr': 9.659468682397744e-05, 'learning_rate': 0.0001929677376394154, 'batch_size': 16, 'weight_decay': 8.89084578924474e-05}. Best is trial 0 with value: 0.8627450980392157.\n",
      "/usr/local/lib/python3.10/dist-packages/accelerate/accelerator.py:488: FutureWarning: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead.\n",
      "  self.scaler = torch.cuda.amp.GradScaler(**kwargs)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='690' max='690' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [690/690 00:49, Epoch 3/3]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Epoch</th>\n",
       "      <th>Training Loss</th>\n",
       "      <th>Validation Loss</th>\n",
       "      <th>Accuracy</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.774076</td>\n",
       "      <td>0.801471</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.601371</td>\n",
       "      <td>0.823529</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>0.185600</td>\n",
       "      <td>0.832180</td>\n",
       "      <td>0.811275</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='26' max='26' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [26/26 00:00]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-07-28 17:25:14,549] Trial 2 finished with value: 0.8112745098039216 and parameters: {'lr': 0.002714209060116254, 'learning_rate': 7.78641364490908e-05, 'batch_size': 16, 'weight_decay': 5.707375868378852e-06}. Best is trial 0 with value: 0.8627450980392157.\n",
      "/usr/local/lib/python3.10/dist-packages/accelerate/accelerator.py:488: FutureWarning: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead.\n",
      "  self.scaler = torch.cuda.amp.GradScaler(**kwargs)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='345' max='345' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [345/345 00:21, Epoch 3/3]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Epoch</th>\n",
       "      <th>Training Loss</th>\n",
       "      <th>Validation Loss</th>\n",
       "      <th>Accuracy</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.798127</td>\n",
       "      <td>0.796569</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.738080</td>\n",
       "      <td>0.811275</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.787325</td>\n",
       "      <td>0.811275</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='13' max='13' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [13/13 00:00]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-07-28 17:25:42,534] Trial 3 finished with value: 0.8112745098039216 and parameters: {'lr': 0.02849462724380318, 'learning_rate': 4.686469948776727e-05, 'batch_size': 32, 'weight_decay': 0.009445506341407504}. Best is trial 0 with value: 0.8627450980392157.\n",
      "/usr/local/lib/python3.10/dist-packages/accelerate/accelerator.py:488: FutureWarning: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead.\n",
      "  self.scaler = torch.cuda.amp.GradScaler(**kwargs)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='174' max='174' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [174/174 00:16, Epoch 3/3]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Epoch</th>\n",
       "      <th>Training Loss</th>\n",
       "      <th>Validation Loss</th>\n",
       "      <th>Accuracy</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.979281</td>\n",
       "      <td>0.808824</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>No log</td>\n",
       "      <td>1.002921</td>\n",
       "      <td>0.784314</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>No log</td>\n",
       "      <td>1.022995</td>\n",
       "      <td>0.794118</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='7' max='7' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [7/7 00:00]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-07-28 17:26:05,829] Trial 4 finished with value: 0.7941176470588235 and parameters: {'lr': 0.00037907572489062966, 'learning_rate': 6.345208973672867e-05, 'batch_size': 64, 'weight_decay': 3.4635045971850106e-06}. Best is trial 0 with value: 0.8627450980392157.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'lr': 1.1338636714975849e-05, 'learning_rate': 0.00011661427090367127, 'batch_size': 32, 'weight_decay': 9.066757241590247e-05}\n"
     ]
    }
   ],
   "source": [
    "import optuna\n",
    "import torch\n",
    "from datasets import load_dataset\n",
    "from transformers import Trainer, TrainingArguments\n",
    "from transformers import BertTokenizer, BertForSequenceClassification\n",
    "import numpy as np\n",
    "\n",
    "model_name = 'bert-base-uncased'\n",
    "tokenizer = BertTokenizer.from_pretrained(model_name)\n",
    "model = BertForSequenceClassification.from_pretrained(model_name)\n",
    "\n",
    "def compute_metrics(eval_pred):\n",
    "    logits, labels = eval_pred\n",
    "    predictions = np.argmax(logits, axis=-1)\n",
    "    return {\"accuracy\": (predictions == labels).mean()} # Define accuracy computation\n",
    "\n",
    "def preprocess(data):\n",
    "   return tokenizer(data['sentence1'], data['sentence2'], truncation=True, padding='max_length', max_length=100)\n",
    "\n",
    "def objective(trial):\n",
    "    lr = trial.suggest_float('lr', 1e-5, 1e-1, log=True)\n",
    "    optimizer = torch.optim.Adam(model.parameters(), lr=lr)\n",
    "\n",
    "    # Load datasets\n",
    "    dataset_train = load_dataset('glue', 'mrpc', split='train')\n",
    "    dataset_validation = load_dataset('glue', 'mrpc', split='validation') # Load validation dataset\n",
    "\n",
    "    # Apply the preprocessing to the datasets\n",
    "    dataset_train = dataset_train.map(preprocess, batched=True)\n",
    "    dataset_validation = dataset_validation.map(preprocess, batched=True) # Preprocess validation dataset\n",
    "\n",
    "    # Remove the unnecessary columns added by the tokenizer\n",
    "    dataset_train = dataset_train.remove_columns(['sentence1', 'sentence2'])\n",
    "    dataset_validation = dataset_validation.remove_columns(['sentence1', 'sentence2']) # Remove columns from validation dataset\n",
    "\n",
    "    # Ensure the format is correct for PyTorch\n",
    "    dataset_train.set_format(type='torch', columns=['input_ids', 'attention_mask', 'label'])\n",
    "    dataset_validation.set_format(type='torch', columns=['input_ids', 'attention_mask', 'label']) # Set format for validation dataset\n",
    "\n",
    "    # Suggest hyperparameters\n",
    "    learning_rate = trial.suggest_float('learning_rate', 1e-5, 1e-3, log=True)\n",
    "    batch_size = trial.suggest_categorical('batch_size', [16, 32, 64])\n",
    "    weight_decay = trial.suggest_float('weight_decay', 1e-6, 1e-2, log=True)\n",
    "\n",
    "    # Define training arguments\n",
    "    training_args = TrainingArguments(\n",
    "        output_dir='./results',\n",
    "        eval_strategy='epoch',\n",
    "        learning_rate=learning_rate,\n",
    "        per_device_train_batch_size=batch_size,\n",
    "        per_device_eval_batch_size=batch_size,\n",
    "        num_train_epochs=3,\n",
    "        weight_decay=weight_decay,\n",
    "    )\n",
    "\n",
    "    # Initialize Trainer\n",
    "    trainer = Trainer(\n",
    "        model=model,\n",
    "        args=training_args,\n",
    "        train_dataset=dataset_train,\n",
    "        eval_dataset=dataset_validation,\n",
    "        compute_metrics=compute_metrics, # Pass the compute_metrics function\n",
    "    )\n",
    "\n",
    "    # Train and evaluate\n",
    "    trainer.train()\n",
    "    eval_result = trainer.evaluate()\n",
    "    return eval_result['eval_accuracy'] # Now eval_result should contain 'eval_accuracy'\n",
    "\n",
    "\n",
    "study = optuna.create_study(direction='maximize')\n",
    "study.optimize(objective, n_trials=5) #recommend 100 or so\n",
    "print(study.best_params)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 198,
   "metadata": {
    "id": "Wqp3-rX_eANu"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Proceed (Y/n)? \u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
      "\u001b[0myes: standard output: Broken pipe\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\r\n",
      "\u001b[0m"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\r\n",
      "\u001b[0m"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\r\n",
      "\u001b[0m"
     ]
    }
   ],
   "source": [
    "!yes|pip uninstall numpy -q\n",
    "!pip install numpy==1.26.0 -q\n",
    "!pip install --upgrade datasets==2.19.2 transformers==4.41.2 -q\n",
    "!pip install pandas==2.0.3 -q\n",
    "# you'll need to restart the runtime after this"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 199,
   "metadata": {
    "id": "3TU5O8FuDx08"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "absl-py==2.1.0\n",
      "accelerate==0.33.0\n",
      "aiohttp==3.9.5\n",
      "aiosignal==1.3.1\n",
      "alembic==1.13.2\n",
      "annotated-types==0.7.0\n",
      "anyio==4.4.0\n",
      "appdirs==1.4.4\n",
      "argon2-cffi==21.1.0\n",
      "astunparse==1.6.3\n",
      "async-timeout==4.0.3\n",
      "attrs==21.2.0\n",
      "Babel==2.8.0\n",
      "backcall==0.2.0\n",
      "beautifulsoup4==4.10.0\n",
      "beniget==0.4.1\n",
      "bleach==4.1.0\n",
      "blinker==1.4\n",
      "Brotli==1.0.9\n",
      "certifi==2024.7.4\n",
      "chardet==4.0.0\n",
      "charset-normalizer==3.3.2\n",
      "colorlog==6.8.2\n",
      "comm==0.2.2\n",
      "command-not-found==0.3\n",
      "cryptography==3.4.8\n",
      "cycler==0.11.0\n",
      "datasets==2.19.2\n",
      "dbus-python==1.2.18\n",
      "decorator==4.4.2\n",
      "defusedxml==0.7.1\n",
      "dill==0.3.8\n",
      "distro==1.7.0\n",
      "distro-info==1.1+ubuntu0.2\n",
      "entrypoints==0.4\n",
      "exceptiongroup==1.2.2\n",
      "filelock==3.15.4\n",
      "flatbuffers==24.3.25\n",
      "fonttools==4.29.1\n",
      "frozenlist==1.4.1\n",
      "fs==2.4.12\n",
      "fsspec==2024.3.1\n",
      "gast==0.6.0\n",
      "google==3.0.0\n",
      "google-pasta==0.2.0\n",
      "greenlet==3.0.3\n",
      "grpcio==1.65.1\n",
      "h11==0.14.0\n",
      "h5py==3.11.0\n",
      "html5lib==1.1\n",
      "httpcore==1.0.5\n",
      "httplib2==0.20.2\n",
      "httpx==0.27.0\n",
      "huggingface-hub==0.24.2\n",
      "idna==3.7\n",
      "importlib-metadata==4.6.4\n",
      "ipykernel==6.7.0\n",
      "ipython==7.31.1\n",
      "ipython_genutils==0.2.0\n",
      "ipywidgets==8.1.3\n",
      "jedi==0.18.0\n",
      "jeepney==0.7.1\n",
      "Jinja2==3.0.3\n",
      "joblib==1.4.2\n",
      "jsonschema==3.2.0\n",
      "jupyter-client==7.1.2\n",
      "jupyter-console==6.4.0\n",
      "jupyter-core==4.9.1\n",
      "jupyterlab-pygments==0.1.2\n",
      "jupyterlab_widgets==3.0.11\n",
      "keras==3.4.1\n",
      "keras-tuner==1.4.7\n",
      "keyring==23.5.0\n",
      "kiwisolver==1.3.2\n",
      "kt-legacy==1.0.5\n",
      "launchpadlib==1.10.16\n",
      "lazr.restfulclient==0.14.4\n",
      "lazr.uri==1.0.6\n",
      "libclang==18.1.1\n",
      "lxml==4.8.0\n",
      "lz4==3.1.3+dfsg\n",
      "Mako==1.3.5\n",
      "Markdown==3.6\n",
      "markdown-it-py==3.0.0\n",
      "MarkupSafe==2.1.5\n",
      "matplotlib==3.5.1\n",
      "matplotlib-inline==0.1.3\n",
      "mdurl==0.1.2\n",
      "ml-dtypes==0.4.0\n",
      "more-itertools==8.10.0\n",
      "mpmath==0.0.0\n",
      "multidict==6.0.5\n",
      "multiprocess==0.70.16\n",
      "namex==0.0.8\n",
      "nbclient==0.5.6\n",
      "nbconvert==6.4.0\n",
      "nbformat==5.1.3\n",
      "nest-asyncio==1.5.4\n",
      "netifaces==0.11.0\n",
      "networkx==3.3\n",
      "notebook==6.4.8\n",
      "numpy==1.26.0\n",
      "nvidia-cublas-cu12==12.1.3.1\n",
      "nvidia-cuda-cupti-cu12==12.1.105\n",
      "nvidia-cuda-nvrtc-cu12==12.1.105\n",
      "nvidia-cuda-runtime-cu12==12.1.105\n",
      "nvidia-cudnn-cu12==9.1.0.70\n",
      "nvidia-cufft-cu12==11.0.2.54\n",
      "nvidia-curand-cu12==10.3.2.106\n",
      "nvidia-cusolver-cu12==11.4.5.107\n",
      "nvidia-cusparse-cu12==12.1.0.106\n",
      "nvidia-nccl-cu12==2.20.5\n",
      "nvidia-nvjitlink-cu12==12.5.82\n",
      "nvidia-nvtx-cu12==12.1.105\n",
      "oauthlib==3.2.0\n",
      "olefile==0.46\n",
      "openai==1.37.1\n",
      "opt-einsum==3.3.0\n",
      "optree==0.12.1\n",
      "optuna==3.6.1\n",
      "packaging==21.3\n",
      "pandas==2.0.3\n",
      "pandocfilters==1.5.0\n",
      "parso==0.8.1\n",
      "pexpect==4.8.0\n",
      "pickleshare==0.7.5\n",
      "Pillow==9.0.1\n",
      "ply==3.11\n",
      "prometheus-client==0.9.0\n",
      "prompt-toolkit==3.0.28\n",
      "protobuf==4.25.4\n",
      "psutil==6.0.0\n",
      "ptyprocess==0.7.0\n",
      "py==1.10.0\n",
      "pyarrow==17.0.0\n",
      "pyarrow-hotfix==0.6\n",
      "pydantic==2.8.2\n",
      "pydantic_core==2.20.1\n",
      "Pygments==2.18.0\n",
      "PyGObject==3.42.1\n",
      "PyJWT==2.3.0\n",
      "pyparsing==2.4.7\n",
      "pyrsistent==0.18.1\n",
      "python-apt==2.4.0+ubuntu3\n",
      "python-dateutil==2.9.0.post0\n",
      "pythran==0.10.0\n",
      "pytz==2022.1\n",
      "PyYAML==5.4.1\n",
      "pyzmq==22.3.0\n",
      "regex==2024.7.24\n",
      "requests==2.32.3\n",
      "rich==13.7.1\n",
      "safetensors==0.4.3\n",
      "scikit-learn==1.5.1\n",
      "scipy==1.8.0\n",
      "screen-resolution-extra==0.0.0\n",
      "SecretStorage==3.3.1\n",
      "Send2Trash==1.8.1b0\n",
      "sentencepiece==0.2.0\n",
      "six==1.16.0\n",
      "sniffio==1.3.1\n",
      "soupsieve==2.3.1\n",
      "SQLAlchemy==2.0.31\n",
      "sympy==1.9\n",
      "systemd-python==234\n",
      "tensorboard==2.17.0\n",
      "tensorboard-data-server==0.7.2\n",
      "tensorflow==2.17.0\n",
      "tensorflow-io-gcs-filesystem==0.37.1\n",
      "termcolor==2.4.0\n",
      "terminado==0.13.1\n",
      "testpath==0.5.0\n",
      "threadpoolctl==3.5.0\n",
      "tokenizers==0.19.1\n",
      "torch==2.4.0\n",
      "torchaudio==2.4.0\n",
      "torchvision==0.19.0\n",
      "tornado==6.1\n",
      "tqdm==4.66.4\n",
      "traitlets==5.1.1\n",
      "transformers==4.41.2\n",
      "triton==3.0.0\n",
      "typing_extensions==4.12.2\n",
      "tzdata==2024.1\n",
      "ubuntu-pro-client==8001\n",
      "ufoLib2==0.13.1\n",
      "ufw==0.36.1\n",
      "unattended-upgrades==0.1\n",
      "unicodedata2==14.0.0\n",
      "urllib3==2.2.2\n",
      "wadllib==1.3.6\n",
      "wcwidth==0.2.5\n",
      "webencodings==0.5.1\n",
      "Werkzeug==3.0.3\n",
      "widgetsnbextension==4.0.11\n",
      "wrapt==1.16.0\n",
      "xkit==0.0.0\n",
      "xxhash==3.4.1\n",
      "yarl==1.9.4\n",
      "zipp==1.0.0\n",
      "Note: you may need to restart the kernel to use updated packages.\n"
     ]
    }
   ],
   "source": [
    "pip freeze"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 200,
   "metadata": {
    "id": "m7LxhjsmV_xa"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\r\n",
      "\u001b[0m"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\r\n",
      "\u001b[0m"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\r\n",
      "\u001b[0m"
     ]
    }
   ],
   "source": [
    "!pip install torch torchvision torchaudio -q\n",
    "!pip install transformers[torch] -q\n",
    "!pip install datasets -q\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 201,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Proceed (Y/n)? \u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
      "\u001b[0myes: standard output: Broken pipe\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\r\n",
      "\u001b[0m"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\r\n",
      "tensorflow 2.17.0 requires numpy<2.0.0,>=1.23.5; python_version <= \"3.11\", but you have numpy 2.0.1 which is incompatible.\r\n",
      "accelerate 0.33.0 requires numpy<2.0.0,>=1.17, but you have numpy 2.0.1 which is incompatible.\u001b[0m\u001b[31m\r\n",
      "\u001b[0m\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\r\n",
      "\u001b[0m"
     ]
    }
   ],
   "source": [
    "!yes|pip uninstall pandas --quiet\n",
    "!pip install pandas --quiet\n",
    "!pip install --upgrade numpy pyarrow --quiet\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 202,
   "metadata": {
    "id": "MrB1UsY_UGol"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of BertForSequenceClassification were not initialized from the model checkpoint at bert-base-uncased and are newly initialized: ['classifier.bias', 'classifier.weight']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    }
   ],
   "source": [
    "from transformers import BertTokenizer, BertForSequenceClassification\n",
    "from datasets import load_dataset\n",
    "\n",
    "# Load pre-trained model and tokenizer\n",
    "model_name = 'bert-base-uncased'\n",
    "tokenizer = BertTokenizer.from_pretrained(model_name)\n",
    "model = BertForSequenceClassification.from_pretrained(model_name)\n",
    "\n",
    "# Load dataset\n",
    "dataset = load_dataset('glue', 'mrpc', split='train')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 203,
   "metadata": {
    "id": "IW5fKmwufxV9"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\r\n",
      "\u001b[0m"
     ]
    }
   ],
   "source": [
    "!pip install accelerate>=0.21.0\n",
    "# you'll need to restart the runtime after this"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 204,
   "metadata": {
    "id": "FCdqiyMsV_9b"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of BertForSequenceClassification were not initialized from the model checkpoint at bert-base-uncased and are newly initialized: ['classifier.bias', 'classifier.weight']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    }
   ],
   "source": [
    "from transformers import BertTokenizer, BertForSequenceClassification\n",
    "from datasets import load_dataset\n",
    "\n",
    "# Load pre-trained model and tokenizer\n",
    "model_name = 'bert-base-uncased'\n",
    "tokenizer = BertTokenizer.from_pretrained(model_name)\n",
    "model = BertForSequenceClassification.from_pretrained(model_name)\n",
    "\n",
    "# Load dataset\n",
    "dataset = load_dataset('glue', 'mrpc', split='train')\n",
    "dataset_validation = load_dataset('glue', 'mrpc', split='validation') # Load validation dataset\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 205,
   "metadata": {
    "id": "1drwHZVuWnKp"
   },
   "outputs": [],
   "source": [
    "def preprocess(data):\n",
    "   return tokenizer(data['sentence1'], data['sentence2'], truncation=True, padding='max_length', max_length=100)\n",
    "\n",
    "dataset = dataset.map(preprocess, batched=True)\n",
    "dataset.set_format(type='torch', columns=['input_ids', 'attention_mask', 'label'])\n",
    "dataset_validation = dataset_validation.map(preprocess, batched=True)\n",
    "dataset_validation.set_format(type='torch', columns=['input_ids', 'attention_mask', 'label']) # Set format for validation dataset#"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 206,
   "metadata": {
    "id": "QPzDb5ILWnVQ"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.10/dist-packages/accelerate/accelerator.py:488: FutureWarning: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead.\n",
      "  self.scaler = torch.cuda.amp.GradScaler(**kwargs)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='1377' max='1377' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [1377/1377 01:47, Epoch 3/3]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Epoch</th>\n",
       "      <th>Training Loss</th>\n",
       "      <th>Validation Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.356673</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>0.509400</td>\n",
       "      <td>0.383166</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>0.315800</td>\n",
       "      <td>0.585230</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "TrainOutput(global_step=1377, training_loss=0.35607646424093364, metrics={'train_runtime': 107.4627, 'train_samples_per_second': 102.398, 'train_steps_per_second': 12.814, 'total_flos': 565483213512000.0, 'train_loss': 0.35607646424093364, 'epoch': 3.0})"
      ]
     },
     "execution_count": 206,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from transformers import Trainer, TrainingArguments\n",
    "\n",
    "training_args = TrainingArguments(\n",
    "   output_dir='./results',\n",
    "   eval_strategy='epoch',\n",
    "   learning_rate=2e-5,\n",
    "   per_device_train_batch_size=8,\n",
    "   per_device_eval_batch_size=8,\n",
    "   num_train_epochs=3,\n",
    "   weight_decay=0.01,\n",
    ")\n",
    "\n",
    "trainer = Trainer(\n",
    "   model=model,\n",
    "   args=training_args,\n",
    "   train_dataset=dataset,\n",
    "   eval_dataset=dataset_validation,\n",
    ")\n",
    "\n",
    "trainer.train()\n",
    "eval_results = trainer.evaluate()\n",
    "print(f\"Evaluation results: {eval_results}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 207,
   "metadata": {
    "id": "E9xTxdbTin5_"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of BertForSequenceClassification were not initialized from the model checkpoint at bert-base-uncased and are newly initialized: ['classifier.bias', 'classifier.weight']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
      "/usr/local/lib/python3.10/dist-packages/accelerate/accelerator.py:488: FutureWarning: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead.\n",
      "  self.scaler = torch.cuda.amp.GradScaler(**kwargs)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='1377' max='1377' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [1377/1377 01:49, Epoch 3/3]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Epoch</th>\n",
       "      <th>Training Loss</th>\n",
       "      <th>Validation Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.370320</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>0.521400</td>\n",
       "      <td>0.359686</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>0.334000</td>\n",
       "      <td>0.545537</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "TrainOutput(global_step=1377, training_loss=0.37095952362969203, metrics={'train_runtime': 109.8975, 'train_samples_per_second': 100.13, 'train_steps_per_second': 12.53, 'total_flos': 565483213512000.0, 'train_loss': 0.37095952362969203, 'epoch': 3.0})"
      ]
     },
     "execution_count": 207,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from transformers import BertTokenizer, BertForSequenceClassification\n",
    "from datasets import load_dataset\n",
    "\n",
    "# Load pre-trained model and tokenizer\n",
    "model_name = 'bert-base-uncased'\n",
    "tokenizer = BertTokenizer.from_pretrained(model_name)\n",
    "model = BertForSequenceClassification.from_pretrained(model_name)\n",
    "\n",
    "# Load datasets\n",
    "dataset_train = load_dataset('glue', 'mrpc', split='train')\n",
    "dataset_validation = load_dataset('glue', 'mrpc', split='validation') # Load validation dataset\n",
    "\n",
    "def preprocess(data):\n",
    "    # Correctly apply padding and return as PyTorch tensors\n",
    "    return tokenizer(data['sentence1'], data['sentence2'], truncation=True, padding='max_length', max_length=100, return_tensors='pt')\n",
    "\n",
    "# Apply the preprocessing to the datasets\n",
    "dataset_train = dataset_train.map(preprocess, batched=True)\n",
    "dataset_validation = dataset_validation.map(preprocess, batched=True) # Preprocess validation dataset\n",
    "\n",
    "# Remove the unnecessary columns added by the tokenizer\n",
    "dataset_train = dataset_train.remove_columns(['sentence1', 'sentence2'])\n",
    "dataset_validation = dataset_validation.remove_columns(['sentence1', 'sentence2']) # Remove columns from validation dataset\n",
    "\n",
    "# Ensure the format is correct for PyTorch\n",
    "dataset_train.set_format(type='torch', columns=['input_ids', 'attention_mask', 'label'])\n",
    "dataset_validation.set_format(type='torch', columns=['input_ids', 'attention_mask', 'label']) # Set format for validation dataset\n",
    "\n",
    "from transformers import Trainer, TrainingArguments\n",
    "\n",
    "# Define training arguments\n",
    "training_args = TrainingArguments(\n",
    "    output_dir='./results',\n",
    "    eval_strategy='epoch',\n",
    "    learning_rate=2e-5,\n",
    "    per_device_train_batch_size=8,\n",
    "    per_device_eval_batch_size=8,\n",
    "    num_train_epochs=3,\n",
    "    weight_decay=0.01,\n",
    ")\n",
    "\n",
    "# Initialize the Trainer\n",
    "trainer = Trainer(\n",
    "    model=model,\n",
    "    args=training_args,\n",
    "    train_dataset=dataset_train, # Pass training dataset\n",
    "    eval_dataset=dataset_validation # Pass validation dataset\n",
    ")\n",
    "\n",
    "# Train the model\n",
    "trainer.train()\n",
    "eval_results = trainer.evaluate()\n",
    "print(f\"Evaluation results: {eval_results}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 208,
   "metadata": {
    "id": "FNr6169TWnf5"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='51' max='51' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [51/51 00:00]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluation results: {'eval_loss': 0.5455366969108582, 'eval_runtime': 0.3651, 'eval_samples_per_second': 1117.472, 'eval_steps_per_second': 139.684, 'epoch': 3.0}\n"
     ]
    }
   ],
   "source": [
    "eval_results = trainer.evaluate()\n",
    "print(f\"Evaluation results: {eval_results}\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 209,
   "metadata": {
    "id": "QXkbe1gXoQ1o"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\r\n",
      "\u001b[0m"
     ]
    }
   ],
   "source": [
    "!pip install optuna --quiet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 210,
   "metadata": {
    "id": "etQeXNjBXutN"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of BertForSequenceClassification were not initialized from the model checkpoint at bert-base-uncased and are newly initialized: ['classifier.bias', 'classifier.weight']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
      "[I 2024-07-28 17:30:20,187] A new study created in memory with name: no-name-f7e031a5-430e-4cfc-a069-9fbb0987616f\n",
      "/usr/local/lib/python3.10/dist-packages/accelerate/accelerator.py:488: FutureWarning: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead.\n",
      "  self.scaler = torch.cuda.amp.GradScaler(**kwargs)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='690' max='690' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [690/690 00:50, Epoch 3/3]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Epoch</th>\n",
       "      <th>Training Loss</th>\n",
       "      <th>Validation Loss</th>\n",
       "      <th>Accuracy</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.401478</td>\n",
       "      <td>0.828431</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.365273</td>\n",
       "      <td>0.835784</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>0.426100</td>\n",
       "      <td>0.512919</td>\n",
       "      <td>0.857843</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='26' max='26' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [26/26 00:00]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-07-28 17:31:17,089] Trial 0 finished with value: 0.8578431372549019 and parameters: {'learning_rate': 3.3766600469032354e-05, 'batch_size': 16, 'weight_decay': 5.655946542236474e-06}. Best is trial 0 with value: 0.8578431372549019.\n",
      "/usr/local/lib/python3.10/dist-packages/accelerate/accelerator.py:488: FutureWarning: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead.\n",
      "  self.scaler = torch.cuda.amp.GradScaler(**kwargs)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='345' max='345' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [345/345 00:22, Epoch 3/3]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Epoch</th>\n",
       "      <th>Training Loss</th>\n",
       "      <th>Validation Loss</th>\n",
       "      <th>Accuracy</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.516834</td>\n",
       "      <td>0.840686</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.540045</td>\n",
       "      <td>0.845588</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.756259</td>\n",
       "      <td>0.825980</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='13' max='13' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [13/13 00:00]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-07-28 17:31:45,642] Trial 1 finished with value: 0.8259803921568627 and parameters: {'learning_rate': 0.00010320747483996698, 'batch_size': 32, 'weight_decay': 0.005292846489554301}. Best is trial 0 with value: 0.8578431372549019.\n",
      "/usr/local/lib/python3.10/dist-packages/accelerate/accelerator.py:488: FutureWarning: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead.\n",
      "  self.scaler = torch.cuda.amp.GradScaler(**kwargs)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='174' max='174' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [174/174 00:17, Epoch 3/3]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Epoch</th>\n",
       "      <th>Training Loss</th>\n",
       "      <th>Validation Loss</th>\n",
       "      <th>Accuracy</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.781096</td>\n",
       "      <td>0.825980</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.684520</td>\n",
       "      <td>0.840686</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.758534</td>\n",
       "      <td>0.855392</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='7' max='7' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [7/7 00:00]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-07-28 17:32:08,959] Trial 2 finished with value: 0.8553921568627451 and parameters: {'learning_rate': 0.00010686519151668729, 'batch_size': 64, 'weight_decay': 0.0001558338855215085}. Best is trial 0 with value: 0.8578431372549019.\n",
      "/usr/local/lib/python3.10/dist-packages/accelerate/accelerator.py:488: FutureWarning: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead.\n",
      "  self.scaler = torch.cuda.amp.GradScaler(**kwargs)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='345' max='345' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [345/345 00:21, Epoch 3/3]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Epoch</th>\n",
       "      <th>Training Loss</th>\n",
       "      <th>Validation Loss</th>\n",
       "      <th>Accuracy</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.749166</td>\n",
       "      <td>0.840686</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.874927</td>\n",
       "      <td>0.816176</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.870012</td>\n",
       "      <td>0.830882</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='13' max='13' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [13/13 00:00]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-07-28 17:32:37,784] Trial 3 finished with value: 0.8308823529411765 and parameters: {'learning_rate': 0.00011763112082047735, 'batch_size': 32, 'weight_decay': 5.6652660608438875e-05}. Best is trial 0 with value: 0.8578431372549019.\n",
      "/usr/local/lib/python3.10/dist-packages/accelerate/accelerator.py:488: FutureWarning: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead.\n",
      "  self.scaler = torch.cuda.amp.GradScaler(**kwargs)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='345' max='345' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [345/345 00:21, Epoch 3/3]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Epoch</th>\n",
       "      <th>Training Loss</th>\n",
       "      <th>Validation Loss</th>\n",
       "      <th>Accuracy</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.650502</td>\n",
       "      <td>0.683824</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.625393</td>\n",
       "      <td>0.683824</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.623878</td>\n",
       "      <td>0.683824</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='13' max='13' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [13/13 00:00]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-07-28 17:33:05,756] Trial 4 finished with value: 0.6838235294117647 and parameters: {'learning_rate': 0.0008782141557764709, 'batch_size': 32, 'weight_decay': 2.5064304690933135e-06}. Best is trial 0 with value: 0.8578431372549019.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best hyperparameters: {'learning_rate': 3.3766600469032354e-05, 'batch_size': 16, 'weight_decay': 5.655946542236474e-06}\n"
     ]
    }
   ],
   "source": [
    "import optuna\n",
    "from transformers import Trainer, TrainingArguments\n",
    "import numpy as np\n",
    "from datasets import load_dataset\n",
    "from transformers import BertTokenizer, BertForSequenceClassification\n",
    "\n",
    "# Load pre-trained model and tokenizer\n",
    "model_name = 'bert-base-uncased'\n",
    "tokenizer = BertTokenizer.from_pretrained(model_name)\n",
    "model = BertForSequenceClassification.from_pretrained(model_name)\n",
    "\n",
    "def compute_metrics(eval_pred):\n",
    "    logits, labels = eval_pred\n",
    "    predictions = np.argmax(logits, axis=-1)\n",
    "    return {\"accuracy\": (predictions == labels).mean()} # Define accuracy computation\n",
    "\n",
    "def preprocess(data):\n",
    "    # Correctly apply padding and return as PyTorch tensors\n",
    "    return tokenizer(data['sentence1'], data['sentence2'], truncation=True, padding='max_length', max_length=100, return_tensors='pt')\n",
    "\n",
    "def objective(trial):\n",
    "    # Load datasets\n",
    "    dataset_train = load_dataset('glue', 'mrpc', split='train')\n",
    "    dataset_validation = load_dataset('glue', 'mrpc', split='validation') # Load validation dataset\n",
    "\n",
    "    # Apply the preprocessing to the datasets\n",
    "    dataset_train = dataset_train.map(preprocess, batched=True)\n",
    "    dataset_validation = dataset_validation.map(preprocess, batched=True) # Preprocess validation dataset\n",
    "\n",
    "    # Remove the unnecessary columns added by the tokenizer\n",
    "    dataset_train = dataset_train.remove_columns(['sentence1', 'sentence2'])\n",
    "    dataset_validation = dataset_validation.remove_columns(['sentence1', 'sentence2']) # Remove columns from validation dataset\n",
    "\n",
    "    # Ensure the format is correct for PyTorch\n",
    "    dataset_train.set_format(type='torch', columns=['input_ids', 'attention_mask', 'label'])\n",
    "    dataset_validation.set_format(type='torch', columns=['input_ids', 'attention_mask', 'label']) # Set format for validation dataset\n",
    "\n",
    "    # Suggest hyperparameters\n",
    "    learning_rate = trial.suggest_float('learning_rate', 1e-5, 1e-3, log=True)\n",
    "    batch_size = trial.suggest_categorical('batch_size', [16, 32, 64])\n",
    "    weight_decay = trial.suggest_float('weight_decay', 1e-6, 1e-2, log=True)\n",
    "\n",
    "    # Define training arguments\n",
    "    training_args = TrainingArguments(\n",
    "        output_dir='./results',\n",
    "        eval_strategy='epoch',\n",
    "        learning_rate=learning_rate,\n",
    "        per_device_train_batch_size=batch_size,\n",
    "        per_device_eval_batch_size=batch_size,\n",
    "        num_train_epochs=3,\n",
    "        weight_decay=weight_decay,\n",
    "    )\n",
    "\n",
    "    # Initialize Trainer\n",
    "    trainer = Trainer(\n",
    "        model=model,\n",
    "        args=training_args,\n",
    "        train_dataset=dataset_train,\n",
    "        eval_dataset=dataset_validation,\n",
    "        compute_metrics=compute_metrics, # Pass the compute_metrics function\n",
    "    )\n",
    "\n",
    "    # Train and evaluate\n",
    "    trainer.train()\n",
    "    eval_result = trainer.evaluate()\n",
    "    return eval_result['eval_accuracy'] # Now eval_result should contain 'eval_accuracy'\n",
    "\n",
    "# Create study and optimize\n",
    "study = optuna.create_study(direction='maximize')\n",
    "study.optimize(objective, n_trials=5) #50 trials would be more useful\n",
    "\n",
    "print(f\"Best hyperparameters: {study.best_params}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "-7uMbKlunU7E"
   },
   "source": [
    "# Section 3.3"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "vTB8ZA9T9Jwk"
   },
   "source": [
    "Working Grid Search Below\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 211,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\r\n",
      "\u001b[0m"
     ]
    }
   ],
   "source": [
    "!pip install scikit-learn --quiet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 212,
   "metadata": {
    "id": "_hjDGI2I3sld"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.10/dist-packages/sklearn/model_selection/_validation.py:540: FitFailedWarning: \n",
      "108 fits failed out of a total of 216.\n",
      "The score on these train-test partitions for these parameters will be set to nan.\n",
      "If these failures are not expected, you can try to debug them by setting error_score='raise'.\n",
      "\n",
      "Below are more details about the failures:\n",
      "--------------------------------------------------------------------------------\n",
      "108 fits failed with the following error:\n",
      "Traceback (most recent call last):\n",
      "  File \"/usr/local/lib/python3.10/dist-packages/sklearn/model_selection/_validation.py\", line 888, in _fit_and_score\n",
      "    estimator.fit(X_train, y_train, **fit_params)\n",
      "  File \"/usr/local/lib/python3.10/dist-packages/sklearn/utils/validation.py\", line 66, in inner_f\n",
      "    return f(*args, **kwargs)\n",
      "  File \"/usr/local/lib/python3.10/dist-packages/sklearn/base.py\", line 1473, in wrapper\n",
      "    return fit_method(estimator, *args, **kwargs)\n",
      "  File \"/usr/local/lib/python3.10/dist-packages/sklearn/ensemble/_bagging.py\", line 402, in fit\n",
      "    return self._fit(X, y, max_samples=self.max_samples, **fit_params)\n",
      "  File \"/usr/local/lib/python3.10/dist-packages/sklearn/ensemble/_bagging.py\", line 502, in _fit\n",
      "    raise ValueError(\"Out of bag estimation only available if bootstrap=True\")\n",
      "ValueError: Out of bag estimation only available if bootstrap=True\n",
      "\n",
      "  warnings.warn(some_fits_failed_message, FitFailedWarning)\n",
      "/usr/local/lib/python3.10/dist-packages/sklearn/model_selection/_search.py:1102: UserWarning: One or more of the test scores are non-finite: [0.49569372 0.52592976 0.52124389 0.49220516 0.52660162 0.52229692\n",
      " 0.53712091 0.52108653 0.53050514 0.54844261 0.57677084 0.57262076\n",
      " 0.58218685 0.58917671 0.58047374 0.57233544 0.58149035 0.58737638\n",
      " 0.64277939 0.65521061 0.65880249 0.6564542  0.67017811 0.67055234\n",
      " 0.67264643 0.67808551 0.68188708 0.54873414 0.58869602 0.58469468\n",
      " 0.57668734 0.58827548 0.58700502 0.5959373  0.59342888 0.59584724\n",
      " 0.65026788 0.64150745 0.65308973 0.6560949  0.66881714 0.67131694\n",
      " 0.66952012 0.68199629 0.6760926  0.74228823 0.74948806 0.75315853\n",
      " 0.77158359 0.76905907 0.76906022 0.7725997  0.77706533 0.77605864\n",
      "        nan        nan        nan        nan        nan        nan\n",
      "        nan        nan        nan        nan        nan        nan\n",
      "        nan        nan        nan        nan        nan        nan\n",
      "        nan        nan        nan        nan        nan        nan\n",
      "        nan        nan        nan        nan        nan        nan\n",
      "        nan        nan        nan        nan        nan        nan\n",
      "        nan        nan        nan        nan        nan        nan\n",
      "        nan        nan        nan        nan        nan        nan\n",
      "        nan        nan        nan        nan        nan        nan]\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best hyperparameters: {'bootstrap': True, 'bootstrap_features': False, 'max_features': 1.0, 'max_samples': 1.0, 'n_estimators': 500}\n"
     ]
    }
   ],
   "source": [
    "from sklearn.base import BaseEstimator\n",
    "from sklearn.model_selection import GridSearchCV, KFold, train_test_split  # Import KFold here\n",
    "from transformers import Trainer, TrainingArguments\n",
    "from sklearn.tree import DecisionTreeRegressor  # Import DecisionTreeRegressor for continuous targets\n",
    "from sklearn.ensemble import BaggingRegressor # Import BaggingRegressor for continuous targets\n",
    "from sklearn.datasets import make_regression\n",
    "\n",
    "# Generate a regression dataset with 1000 samples and 10 features\n",
    "X, y = make_regression(n_samples=1000, n_features=10, noise=0.1, random_state=42)\n",
    "\n",
    "X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.2, random_state=42)\n",
    "\n",
    "param_grid = {\n",
    "    'n_estimators': [100, 500, 1000],\n",
    "    'max_samples': [0.5, 0.75, 1.0],\n",
    "    'max_features': [0.5, 0.75, 1.0],\n",
    "    'bootstrap': [True, False],\n",
    "    'bootstrap_features': [True, False]\n",
    "}\n",
    "# Use DecisionTreeRegressor and BaggingRegressor for continuous target\n",
    "clf = DecisionTreeRegressor(criterion='squared_error', max_features='sqrt')\n",
    "classifier = BaggingRegressor(estimator=clf, n_estimators=2000, max_features=1.0,\n",
    "                         max_samples=.5, oob_score=True, n_jobs=1)\n",
    "\n",
    "# Define a 2-fold cross-validator\n",
    "cv = KFold(n_splits=2, shuffle=True, random_state=42)\n",
    "\n",
    "# Use GridSearchCV with the custom estimator\n",
    "grid_search = GridSearchCV(\n",
    "    estimator=classifier,  # Pass your custom estimator\n",
    "    param_grid=param_grid,\n",
    "    # Removed scoring parameter since the estimator now has a score method\n",
    "    refit=True,\n",
    "    cv=cv\n",
    ")\n",
    "\n",
    "# Pass the param_grid to fit, as it contains the hyperparameters to search\n",
    "grid_search.fit(X_train, y_train)\n",
    "\n",
    "# Get the best hyperparameters\n",
    "best_params = grid_search.best_params_\n",
    "print(f\"Best hyperparameters: {best_params}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 213,
   "metadata": {
    "id": "0Ko8-yxr6BMc"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best hyperparameters: {'bootstrap': True, 'bootstrap_features': False, 'max_features': 1.0, 'max_samples': 1.0, 'n_estimators': 500}\n"
     ]
    }
   ],
   "source": [
    "# Get the best hyperparameters\n",
    "best_params = grid_search.best_params_\n",
    "print(f\"Best hyperparameters: {best_params}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 214,
   "metadata": {
    "id": "EWBD0pq9X1uW"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best hyperparameters: {'bootstrap': True, 'bootstrap_features': False, 'max_features': 1.0, 'max_samples': 1.0, 'n_estimators': 500}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.10/dist-packages/sklearn/model_selection/_validation.py:540: FitFailedWarning: \n",
      "25 fits failed out of a total of 50.\n",
      "The score on these train-test partitions for these parameters will be set to nan.\n",
      "If these failures are not expected, you can try to debug them by setting error_score='raise'.\n",
      "\n",
      "Below are more details about the failures:\n",
      "--------------------------------------------------------------------------------\n",
      "25 fits failed with the following error:\n",
      "Traceback (most recent call last):\n",
      "  File \"/usr/local/lib/python3.10/dist-packages/sklearn/model_selection/_validation.py\", line 888, in _fit_and_score\n",
      "    estimator.fit(X_train, y_train, **fit_params)\n",
      "  File \"/usr/local/lib/python3.10/dist-packages/sklearn/utils/validation.py\", line 66, in inner_f\n",
      "    return f(*args, **kwargs)\n",
      "  File \"/usr/local/lib/python3.10/dist-packages/sklearn/base.py\", line 1473, in wrapper\n",
      "    return fit_method(estimator, *args, **kwargs)\n",
      "  File \"/usr/local/lib/python3.10/dist-packages/sklearn/ensemble/_bagging.py\", line 402, in fit\n",
      "    return self._fit(X, y, max_samples=self.max_samples, **fit_params)\n",
      "  File \"/usr/local/lib/python3.10/dist-packages/sklearn/ensemble/_bagging.py\", line 502, in _fit\n",
      "    raise ValueError(\"Out of bag estimation only available if bootstrap=True\")\n",
      "ValueError: Out of bag estimation only available if bootstrap=True\n",
      "\n",
      "  warnings.warn(some_fits_failed_message, FitFailedWarning)\n",
      "/usr/local/lib/python3.10/dist-packages/sklearn/model_selection/_search.py:1102: UserWarning: One or more of the test scores are non-finite: [           nan -6984.41256452 -7914.02953098            nan\n",
      "            nan            nan -6550.05420156 -3802.60841081\n",
      "            nan -7043.63985934]\n",
      "  warnings.warn(\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import RandomizedSearchCV\n",
    "from sklearn.base import BaseEstimator\n",
    "from sklearn.model_selection import KFold, train_test_split  # Import KFold here\n",
    "from transformers import Trainer, TrainingArguments\n",
    "from sklearn.tree import DecisionTreeRegressor  # Import DecisionTreeRegressor for continuous targets\n",
    "from sklearn.ensemble import BaggingRegressor # Import BaggingRegressor for continuous targets\n",
    "from scipy.stats import randint\n",
    "# ... (your existing code for param_grid, train_and_evaluate)\n",
    "from sklearn.datasets import make_regression\n",
    "\n",
    "# Generate a regression dataset with 1000 samples and 10 features\n",
    "X, y = make_regression(n_samples=1000, n_features=10, noise=0.1, random_state=42)\n",
    "\n",
    "X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.2, random_state=42)\n",
    "\n",
    "param_distributions = {\n",
    "    'n_estimators': [100, 500, 1000],\n",
    "    'max_samples': [0.5, 0.75, 1.0],\n",
    "    'max_features': [0.5, 0.75, 1.0],\n",
    "    'bootstrap': [True, False],\n",
    "    'bootstrap_features': [True, False]\n",
    "}\n",
    "\n",
    "# Use DecisionTreeRegressor and BaggingRegressor for continuous target\n",
    "clf = DecisionTreeRegressor(criterion='squared_error', max_features='sqrt')\n",
    "classifier = BaggingRegressor(estimator=clf, n_estimators=1000, max_features=1.,\n",
    "                         max_samples=.5, oob_score=True, n_jobs=1)\n",
    "\n",
    "random_search = RandomizedSearchCV(estimator=classifier, param_distributions=param_distributions, n_iter=10, scoring='neg_mean_squared_error',cv=5, random_state=42)\n",
    "random_search.fit(X_train, y_train)\n",
    "best_params = random_search.best_params_\n",
    "\n",
    "# Get the best hyperparameters\n",
    "best_params = grid_search.best_params_\n",
    "print(f\"Best hyperparameters: {best_params}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 215,
   "metadata": {
    "id": "VpZgFD8bqg-f"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\r\n",
      "\u001b[0m"
     ]
    }
   ],
   "source": [
    "!pip install optuna --quiet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 216,
   "metadata": {
    "id": "yZVBcROzuYsQ"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mean Squared Error: 12561.024266819304\n",
      "R^2 Score: 0.25547221402319586\n",
      "BaggingRegressor(estimator=DecisionTreeRegressor(criterion='absolute_error',\n",
      "                                                 max_depth=10),\n",
      "                 max_samples=5, n_estimators=1582, n_jobs=1, oob_score=True) 12561.024266819304 0.25547221402319586\n"
     ]
    }
   ],
   "source": [
    "# Example usage with RandomizedSearchCV\n",
    "from sklearn.metrics import mean_squared_error, r2_score\n",
    "from sklearn.model_selection import RandomizedSearchCV, train_test_split\n",
    "from sklearn.tree import DecisionTreeRegressor\n",
    "from sklearn.ensemble import BaggingRegressor\n",
    "from sklearn.datasets import make_regression\n",
    "from scipy.stats import randint\n",
    "\n",
    "def train_and_evaluate_model(model, X_train, y_train, X_val, y_val):\n",
    "    # Train the model\n",
    "    model.fit(X_train, y_train)\n",
    "\n",
    "    # Predict on validation set\n",
    "    y_pred = model.predict(X_val)\n",
    "\n",
    "    # Calculate evaluation metrics\n",
    "    mse = mean_squared_error(y_val, y_pred)\n",
    "    r2 = r2_score(y_val, y_pred)\n",
    "\n",
    "    print(f\"Mean Squared Error: {mse}\")\n",
    "    print(f\"R^2 Score: {r2}\")\n",
    "\n",
    "    return model, mse, r2\n",
    "\n",
    "# Generate a regression dataset with 1000 samples and 10 features\n",
    "X, y = make_regression(n_samples=1000, n_features=10, noise=0.1, random_state=42)\n",
    "X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.2, random_state=42)\n",
    "\n",
    "# Define the parameter distributions\n",
    "param_distributions = {\n",
    "    'estimator__criterion': ['squared_error', 'friedman_mse', 'absolute_error'],\n",
    "    'estimator__max_depth': [None, 5, 10],\n",
    "    'n_estimators': randint(100, 2000)  # Using a range for n_estimators\n",
    "}\n",
    "\n",
    "# Initialize the base estimator and the BaggingRegressor\n",
    "clf = DecisionTreeRegressor()\n",
    "classifier = BaggingRegressor(estimator=clf, n_estimators=1000, max_features=1.0,\n",
    "                              max_samples=5, oob_score=True, n_jobs=1)\n",
    "\n",
    "# Create the RandomizedSearchCV object\n",
    "random_search = RandomizedSearchCV(estimator=classifier, param_distributions=param_distributions, n_iter=10, scoring='neg_mean_squared_error', cv=5, random_state=42)\n",
    "\n",
    "# Fit the model using RandomizedSearchCV\n",
    "random_search.fit(X_train, y_train)\n",
    "\n",
    "# Get the best model from random search\n",
    "best_model = random_search.best_estimator_\n",
    "\n",
    "# Train and evaluate the best model\n",
    "best_model, mse, r2 = train_and_evaluate_model(best_model, X_train, y_train, X_val, y_val)\n",
    "\n",
    "print(best_model, mse, r2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 217,
   "metadata": {
    "id": "Ck0FNe91u-Q8"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\r\n",
      "\u001b[0m"
     ]
    }
   ],
   "source": [
    "!pip install keras keras_tuner -q"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 218,
   "metadata": {
    "id": "zVk1_om0pP7o"
   },
   "outputs": [],
   "source": [
    "import torch.nn as nn\n",
    "\n",
    "# Define a simple neural network model\n",
    "class SimpleCNN(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(SimpleCNN, self).__init__()\n",
    "        self.conv1 = nn.Conv2d(1, 32, kernel_size=3, stride=1, padding=1)\n",
    "        self.conv2 = nn.Conv2d(32, 64, kernel_size=3, stride=1, padding=1)\n",
    "        self.fc1 = nn.Linear(64*7*7, 128)\n",
    "        self.fc2 = nn.Linear(128, 10)\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = torch.relu(self.conv1(x))\n",
    "        x = torch.max_pool2d(x, 2)\n",
    "        x = torch.relu(self.conv2(x))\n",
    "        x = torch.max_pool2d(x, 2)\n",
    "        x = x.view(-1, 64*7*7)\n",
    "        x = torch.relu(self.fc1(x))\n",
    "        x = self.fc2(x)\n",
    "        return x\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 219,
   "metadata": {
    "id": "Zqigf_vO7FBY"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\r\n",
      "\u001b[0m"
     ]
    }
   ],
   "source": [
    "!pip install keras keras-tuner tensorflow --quiet\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 220,
   "metadata": {
    "id": "nj5ifEjAZIrk"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reloading Tuner from my_dir/helloworld/tuner0.json\n",
      "<keras_tuner.src.engine.hyperparameters.hyperparameters.HyperParameters object at 0x7f659040aa40>\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "from tensorflow.keras import layers, models\n",
    "import keras_tuner as kt\n",
    "\n",
    "def SimpleCNN():\n",
    "    model = models.Sequential([\n",
    "        layers.Conv2D(32, (3, 3), activation='relu', input_shape=(28, 28, 1)),\n",
    "        layers.MaxPooling2D((2, 2)),\n",
    "        layers.Conv2D(64, (3, 3), activation='relu'),\n",
    "        layers.MaxPooling2D((2, 2)),\n",
    "        layers.Conv2D(64, (3, 3), activation='relu'),\n",
    "        layers.Flatten(),\n",
    "        layers.Dense(64, activation='relu'),\n",
    "        layers.Dense(10, activation='softmax')\n",
    "    ])\n",
    "    return model\n",
    "\n",
    "\n",
    "def create_model(hp):\n",
    "    model = SimpleCNN()\n",
    "    model.compile(\n",
    "        optimizer=tf.keras.optimizers.Adam(learning_rate=hp.Float('learning_rate', 1e-4, 1e-2, sampling='log')),\n",
    "        loss='sparse_categorical_crossentropy',\n",
    "        metrics=['accuracy']\n",
    "    )\n",
    "\n",
    "    return model\n",
    "\n",
    "tuner = kt.Hyperband(\n",
    "    create_model,\n",
    "    objective='val_accuracy',\n",
    "    max_epochs=5, #often more like 10\n",
    "    hyperband_iterations=1, #often more like 3\n",
    "    factor=3,\n",
    "    directory='my_dir',\n",
    "    project_name='helloworld'\n",
    ")\n",
    "\n",
    "# Example data (replace with your actual data)\n",
    "(X_train, y_train), (X_val, y_val) = tf.keras.datasets.mnist.load_data()\n",
    "X_train = X_train.reshape(-1, 28, 28, 1).astype('float32') / 255.0\n",
    "X_val = X_val.reshape(-1, 28, 28, 1).astype('float32') / 255.0\n",
    "\n",
    "# Run the hyperparameter search\n",
    "tuner.search(X_train, y_train, epochs=10, validation_data=(X_val, y_val))\n",
    "\n",
    "# Get the best hyperparameters\n",
    "best_params = tuner.get_best_hyperparameters(num_trials=1)[0]\n",
    "print(best_params)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "from transformers import BertModel, BertTokenizer, Trainer, TrainingArguments\n",
    "from datasets import load_dataset\n",
    "from transformers import EarlyStoppingCallback, default_data_collator\n",
    "\n",
    "# Custom BERT-based model with additional dropout layers\n",
    "class CustomBertModel(nn.Module):\n",
    "    def __init__(self, model_name='bert-base-uncased', dropout_prob=0.3):\n",
    "        super(CustomBertModel, self).__init__()\n",
    "        self.bert = BertModel.from_pretrained(model_name)\n",
    "        self.dropout = nn.Dropout(dropout_prob)  # Dropout rate is set here\n",
    "        self.classifier = nn.Linear(self.bert.config.hidden_size, 2)  # Assuming binary classification\n",
    "\n",
    "    def forward(self, input_ids, attention_mask):\n",
    "        outputs = self.bert(input_ids=input_ids, attention_mask=attention_mask)\n",
    "        pooled_output = outputs[1]  # The pooled output from the BERT model\n",
    "        pooled_output = self.dropout(pooled_output)  # Apply dropout\n",
    "        logits = self.classifier(pooled_output)\n",
    "        return logits\n",
    "\n",
    "# Load tokenizer\n",
    "tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')\n",
    "\n",
    "# Load datasets\n",
    "dataset_train = load_dataset('glue', 'mrpc', split='train')\n",
    "dataset_val = load_dataset('glue', 'mrpc', split='validation')\n",
    "\n",
    "# Tokenize the datasets\n",
    "def preprocess(data):\n",
    "    return tokenizer(data['sentence1'], data['sentence2'], truncation=True, padding='max_length', max_length=100)\n",
    "\n",
    "dataset_train = dataset_train.map(preprocess, batched=True)\n",
    "dataset_val = dataset_val.map(preprocess, batched=True)\n",
    "\n",
    "dataset_train.set_format(type='torch', columns=['input_ids', 'attention_mask', 'label'])\n",
    "dataset_val.set_format(type='torch', columns=['input_ids', 'attention_mask', 'label'])\n",
    "\n",
    "# Initialize custom model with specified dropout rate\n",
    "model = CustomBertModel(dropout_prob=0.3).to(torch.device('cuda' if torch.cuda.is_available() else 'cpu'))\n",
    "\n",
    "# Define training arguments\n",
    "training_args = TrainingArguments(\n",
    "    output_dir='./results',\n",
    "    evaluation_strategy='epoch',\n",
    "    learning_rate=2e-5,\n",
    "    per_device_train_batch_size=16,\n",
    "    per_device_eval_batch_size=16,\n",
    "    num_train_epochs=3,\n",
    "    weight_decay=0.01,\n",
    "    logging_dir='./logs',\n",
    "    logging_steps=10,\n",
    "    save_steps=10,\n",
    "    load_best_model_at_end=True,\n",
    "    fp16=True  # Enable mixed-precision training\n",
    ")\n",
    "\n",
    "# Define the Trainer\n",
    "trainer = Trainer(\n",
    "    model=model,\n",
    "    args=training_args,\n",
    "    train_dataset=dataset_train,\n",
    "    eval_dataset=dataset_val,\n",
    "    data_collator=default_data_collator,\n",
    "    callbacks=[EarlyStoppingCallback(early_stopping_patience=3)]\n",
    ")\n",
    "\n",
    "# Train the model\n",
    "trainer.train()\n",
    "\n",
    "# Evaluate the model\n",
    "eval_results = trainer.evaluate()\n",
    "print(f\"Evaluation results: {eval_results}\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "9cVz65zYE76E"
   },
   "source": [
    "# Section 3.4 Principles of PEFT (Parameter Efficient Fine-Tuning)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "mjto5sZAV0k-"
   },
   "source": [
    "Adapter Modules"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 221,
   "metadata": {
    "id": "8jFt0w_eXxM8"
   },
   "outputs": [],
   "source": [
    "from transformers import BertModel, BertConfig\n",
    "from torch import nn\n",
    "\n",
    "class Adapter(nn.Module):\n",
    "   def __init__(self, input_dim, bottleneck_dim):\n",
    "       super(Adapter, self).__init__()\n",
    "       self.down_project = nn.Linear(input_dim, bottleneck_dim)\n",
    "       self.up_project = nn.Linear(bottleneck_dim, input_dim)\n",
    "       self.activation = nn.ReLU()\n",
    "\n",
    "   def forward(self, x):\n",
    "       return self.up_project(self.activation(self.down_project(x)))\n",
    "\n",
    "class BertWithAdapters(nn.Module):\n",
    "   def __init__(self, model_name, bottleneck_dim):\n",
    "       super(BertWithAdapters, self).__init__()\n",
    "       self.bert = BertModel.from_pretrained(model_name)\n",
    "       self.adapters = nn.ModuleList([Adapter(self.bert.config.hidden_size, bottleneck_dim) for _ in range(self.bert.config.num_hidden_layers)])\n",
    "\n",
    "   def forward(self, input_ids, attention_mask=None):\n",
    "       outputs = self.bert(input_ids, attention_mask=attention_mask)\n",
    "       for i, adapter in enumerate(self.adapters):\n",
    "          outputs.last_hidden_state[:, i, :] = adapter(outputs.last_hidden_state[:, i, :])\n",
    "          return outputs\n",
    "\n",
    "model_name = 'bert-base-uncased'\n",
    "bottleneck_dim = 64\n",
    "model = BertWithAdapters(model_name, bottleneck_dim)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "9MH9PF8jV8Kv"
   },
   "source": [
    "Layer Freezing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 222,
   "metadata": {
    "id": "oQA3PKbiZgXX"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of BertForSequenceClassification were not initialized from the model checkpoint at bert-base-uncased and are newly initialized: ['classifier.bias', 'classifier.weight']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "from transformers import BertForSequenceClassification\n",
    "\n",
    "model = BertForSequenceClassification.from_pretrained('bert-base-uncased')\n",
    "\n",
    "# Freeze all layers except the classifier\n",
    "for param in model.bert.parameters():\n",
    "   param.requires_grad = False\n",
    "\n",
    "# Fine-tune the classifier layers\n",
    "optimizer = torch.optim.AdamW(model.classifier.parameters(), lr=2e-5)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "UoYHPEovWGMn"
   },
   "source": [
    "L2 Regularization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 223,
   "metadata": {
    "id": "elXP-OQtZgdG"
   },
   "outputs": [],
   "source": [
    "from torch.optim import AdamW\n",
    "# Define optimizer with L2 regularization (weight decay)\n",
    "optimizer = AdamW(model.parameters(), lr=2e-5, weight_decay=0.01)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "KWllpf-8WMAM"
   },
   "source": [
    "Gradient Scaling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 224,
   "metadata": {
    "id": "CLsNnqBbWLj7"
   },
   "outputs": [],
   "source": [
    "optimizer = torch.optim.AdamW([\n",
    "   {'params': model.bert.encoder.layer[-1].parameters(), 'lr': 5e-5},\n",
    "   {'params': model.bert.encoder.layer[-2].parameters(), 'lr': 5e-5},\n",
    "   {'params': model.classifier.parameters(), 'lr': 2e-5}\n",
    "])\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "YYuTMy4hXjdB"
   },
   "source": [
    "Fine-tuning with adapters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 225,
   "metadata": {
    "id": "yUKZTVJexRb6"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\r\n",
      "\u001b[0m"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\r\n",
      "\u001b[0m"
     ]
    }
   ],
   "source": [
    "!pip install accelerate>=0.21.0 -q\n",
    "!pip install datasets -q\n",
    "# you'll need to restart the runtime after this"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 226,
   "metadata": {
    "id": "r9Uf0Qjnzj2L"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\r\n",
      "\u001b[0m"
     ]
    }
   ],
   "source": [
    "!pip install datasets -q"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 227,
   "metadata": {
    "id": "1U-acTO7XiOo"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.10/dist-packages/accelerate/accelerator.py:488: FutureWarning: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead.\n",
      "  self.scaler = torch.cuda.amp.GradScaler(**kwargs)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='690' max='690' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [690/690 00:52, Epoch 3/3]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Epoch</th>\n",
       "      <th>Training Loss</th>\n",
       "      <th>Validation Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.390525</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.355074</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>0.415400</td>\n",
       "      <td>0.439589</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='26' max='26' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [26/26 00:00]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluation results for task 2: {'eval_loss': 0.43958887457847595, 'eval_runtime': 0.229, 'eval_samples_per_second': 1781.271, 'eval_steps_per_second': 113.512, 'epoch': 3.0}\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "from transformers import BertForSequenceClassification, BertModel, BertTokenizer\n",
    "from transformers import Trainer, TrainingArguments\n",
    "from datasets import load_dataset\n",
    "\n",
    "# Define your custom model\n",
    "class BertWithAdapters(torch.nn.Module):\n",
    "    def __init__(self, model_name, bottleneck_dim):\n",
    "        super().__init__()\n",
    "        self.bert = BertModel.from_pretrained(model_name)\n",
    "        self.tokenizer = BertTokenizer.from_pretrained(model_name) # Initialize the tokenizer here\n",
    "        self.bottleneck = torch.nn.Linear(self.bert.config.hidden_size, bottleneck_dim)\n",
    "        self.classifier = torch.nn.Linear(bottleneck_dim, 2) # Assuming binary classification\n",
    "\n",
    "    def forward(self, input_ids, attention_mask, labels=None): # Add labels as argument\n",
    "        outputs = self.bert(input_ids=input_ids, attention_mask=attention_mask)\n",
    "        pooled_output = outputs.pooler_output\n",
    "        bottleneck_output = self.bottleneck(pooled_output)\n",
    "        logits = self.classifier(bottleneck_output)\n",
    "\n",
    "        loss = None\n",
    "        if labels is not None: # Calculate loss if labels are provided\n",
    "            loss_fct = torch.nn.CrossEntropyLoss()\n",
    "            loss = loss_fct(logits.view(-1, 2), labels.view(-1)) # Assuming binary classification\n",
    "\n",
    "        return {'loss': loss, 'logits': logits} # Return loss and logits\n",
    "\n",
    "\n",
    "training_args = TrainingArguments(\n",
    "    output_dir='./results',\n",
    "    eval_strategy='epoch',\n",
    "    learning_rate=2e-5,\n",
    "    per_device_train_batch_size=16,\n",
    "    per_device_eval_batch_size=16,\n",
    "    num_train_epochs=3,\n",
    "    weight_decay=0.01,\n",
    "    logging_dir='./logs',\n",
    ")\n",
    "\n",
    "model_name = 'bert-base-uncased'\n",
    "bottleneck_dim = 64\n",
    "model = BertWithAdapters(model_name, bottleneck_dim)\n",
    "\n",
    "# Load both training and validation datasets\n",
    "dataset_train = load_dataset('glue', 'mrpc', split='train')\n",
    "dataset_val = load_dataset('glue', 'mrpc', split='validation') # Load the validation set\n",
    "\n",
    "# Tokenize the datasets\n",
    "def preprocess_function(examples):\n",
    "    return model.tokenizer(examples['sentence1'], examples['sentence2'], truncation=True, padding=True)\n",
    "\n",
    "dataset_train = dataset_train.map(preprocess_function, batched=True)\n",
    "dataset_val = dataset_val.map(preprocess_function, batched=True) # Tokenize the validation set\n",
    "\n",
    "# Add a data collator to prepare the inputs for the model\n",
    "from transformers import DataCollatorWithPadding\n",
    "data_collator = DataCollatorWithPadding(tokenizer=model.tokenizer) # Use the tokenizer associated with your BERT model\n",
    "\n",
    "trainer = Trainer(\n",
    "    model=model,\n",
    "    args=training_args,\n",
    "    train_dataset=dataset_train, # Pass the training dataset\n",
    "    eval_dataset=dataset_val,   # Pass the validation dataset\n",
    "    data_collator=data_collator # Pass the data collator to the Trainer\n",
    ")\n",
    "\n",
    "trainer.train()\n",
    "eval_results = trainer.evaluate()\n",
    "print(f\"Evaluation results for task {i+1}: {eval_results}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "UyDtqn2yXnuW"
   },
   "source": [
    "Fine-tuning with Layer-freezing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 228,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\r\n",
      "\u001b[0m"
     ]
    }
   ],
   "source": [
    "!pip install datasets transformers --quiet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 229,
   "metadata": {
    "id": "1RxMjzbkXngP"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of BertForSequenceClassification were not initialized from the model checkpoint at bert-base-uncased and are newly initialized: ['classifier.bias', 'classifier.weight']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
      "/usr/local/lib/python3.10/dist-packages/accelerate/accelerator.py:488: FutureWarning: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead.\n",
      "  self.scaler = torch.cuda.amp.GradScaler(**kwargs)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='690' max='690' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [690/690 00:45, Epoch 3/3]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Epoch</th>\n",
       "      <th>Training Loss</th>\n",
       "      <th>Validation Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.467886</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.385722</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>0.460600</td>\n",
       "      <td>0.419906</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='26' max='26' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [26/26 00:00]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluation results for task 2: {'eval_loss': 0.4199060797691345, 'eval_runtime': 0.2414, 'eval_samples_per_second': 1690.468, 'eval_steps_per_second': 107.726, 'epoch': 3.0}\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "from transformers import BertForSequenceClassification, BertModel, BertTokenizer, Trainer, TrainingArguments\n",
    "from datasets import load_dataset\n",
    "\n",
    "# Tokenize the datasets\n",
    "def preprocess_function(examples):\n",
    "    return tokenizer(\n",
    "        examples['sentence1'],\n",
    "        examples['sentence2'],\n",
    "        truncation=True,\n",
    "        return_overflowing_tokens=False, \n",
    "        padding='max_length',  # Add padding to max_length\n",
    "        max_length=96          # Set the maximum sequence length\n",
    "    )\n",
    "\n",
    "model = BertForSequenceClassification.from_pretrained('bert-base-uncased')\n",
    "tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') # Initialize the tokenizer\n",
    "\n",
    "# Only freeze the embeddings\n",
    "for param in model.bert.embeddings.parameters():\n",
    "    param.requires_grad = False\n",
    "\n",
    "training_args = TrainingArguments(\n",
    "   output_dir='./results',\n",
    "   eval_strategy='epoch',\n",
    "   learning_rate=2e-5,\n",
    "   per_device_train_batch_size=16,\n",
    "   per_device_eval_batch_size=16,\n",
    "   num_train_epochs=3,\n",
    "   weight_decay=0.01,\n",
    "   logging_dir='./logs',\n",
    ")\n",
    "\n",
    "# Load both training and validation datasets\n",
    "dataset_train = load_dataset('glue', 'mrpc', split='train')\n",
    "dataset_val = load_dataset('glue', 'mrpc', split='validation') # Load the validation set\n",
    "\n",
    "\n",
    "# Apply the preprocessing function to the datasets\n",
    "dataset_train = dataset_train.map(preprocess_function, batched=True)\n",
    "dataset_val = dataset_val.map(preprocess_function, batched=True)\n",
    "\n",
    "trainer = Trainer(\n",
    "   model=model,\n",
    "   args=training_args,\n",
    "   train_dataset=dataset_train, # Pass the training dataset\n",
    "   eval_dataset=dataset_val,   # Pass the validation dataset\n",
    ")\n",
    "\n",
    "trainer.train()\n",
    "eval_results = trainer.evaluate()\n",
    "print(f\"Evaluation results for task {i+1}: {eval_results}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 230,
   "metadata": {
    "id": "vQctkdHe0yy6"
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2307893532864b97b5acbc6d48a5ace9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Map:   0%|          | 0/3668 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3ff9e33fc5af4d17b979db4fbc0cea69",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Map:   0%|          | 0/408 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.10/dist-packages/accelerate/accelerator.py:488: FutureWarning: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead.\n",
      "  self.scaler = torch.cuda.amp.GradScaler(**kwargs)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='690' max='690' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [690/690 00:52, Epoch 3/3]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Epoch</th>\n",
       "      <th>Training Loss</th>\n",
       "      <th>Validation Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.390073</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.371923</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>0.418800</td>\n",
       "      <td>0.426819</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='26' max='26' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [26/26 00:00]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluation results for task 2: {'eval_loss': 0.4268192946910858, 'eval_runtime': 0.224, 'eval_samples_per_second': 1821.524, 'eval_steps_per_second': 116.077, 'epoch': 3.0}\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "from transformers import BertForSequenceClassification, BertModel, BertTokenizer\n",
    "from transformers import Trainer, TrainingArguments\n",
    "from datasets import load_dataset\n",
    "\n",
    "# Define your custom model\n",
    "class BertWithAdapters(torch.nn.Module):\n",
    "    def __init__(self, model_name, bottleneck_dim):\n",
    "        super().__init__()\n",
    "        self.bert = BertModel.from_pretrained(model_name)\n",
    "        self.tokenizer = BertTokenizer.from_pretrained(model_name) # Initialize the tokenizer here\n",
    "        self.bottleneck = torch.nn.Linear(self.bert.config.hidden_size, bottleneck_dim)\n",
    "        self.classifier = torch.nn.Linear(bottleneck_dim, 2) # Assuming binary classification\n",
    "\n",
    "    def forward(self, input_ids, attention_mask, labels=None): # Add labels as argument\n",
    "        outputs = self.bert(input_ids=input_ids, attention_mask=attention_mask)\n",
    "        pooled_output = outputs.pooler_output\n",
    "        bottleneck_output = self.bottleneck(pooled_output)\n",
    "        logits = self.classifier(bottleneck_output)\n",
    "\n",
    "        loss = None\n",
    "        if labels is not None: # Calculate loss if labels are provided\n",
    "            loss_fct = torch.nn.CrossEntropyLoss()\n",
    "            loss = loss_fct(logits.view(-1, 2), labels.view(-1)) # Assuming binary classification\n",
    "\n",
    "        return {'loss': loss, 'logits': logits} # Return loss and logits\n",
    "\n",
    "\n",
    "training_args = TrainingArguments(\n",
    "    output_dir='./results',\n",
    "    eval_strategy='epoch',\n",
    "    learning_rate=2e-5,\n",
    "    per_device_train_batch_size=16,\n",
    "    per_device_eval_batch_size=16,\n",
    "    num_train_epochs=3,\n",
    "    weight_decay=0.01,\n",
    "    logging_dir='./logs',\n",
    ")\n",
    "\n",
    "model_name = 'bert-base-uncased'\n",
    "bottleneck_dim = 64\n",
    "model = BertWithAdapters(model_name, bottleneck_dim)\n",
    "\n",
    "# Load both training and validation datasets\n",
    "dataset_train = load_dataset('glue', 'mrpc', split='train')\n",
    "dataset_val = load_dataset('glue', 'mrpc', split='validation') # Load the validation set\n",
    "\n",
    "# Tokenize the datasets\n",
    "def preprocess_function(examples):\n",
    "    return model.tokenizer(examples['sentence1'], examples['sentence2'], truncation=True, padding=True)\n",
    "\n",
    "dataset_train = dataset_train.map(preprocess_function, batched=True)\n",
    "dataset_val = dataset_val.map(preprocess_function, batched=True) # Tokenize the validation set\n",
    "\n",
    "# Add a data collator to prepare the inputs for the model\n",
    "from transformers import DataCollatorWithPadding\n",
    "data_collator = DataCollatorWithPadding(tokenizer=model.tokenizer) # Use the tokenizer associated with your BERT model\n",
    "\n",
    "trainer = Trainer(\n",
    "    model=model,\n",
    "    args=training_args,\n",
    "    train_dataset=dataset_train, # Pass the training dataset\n",
    "    eval_dataset=dataset_val,   # Pass the validation dataset\n",
    "    data_collator=data_collator # Pass the data collator to the Trainer\n",
    ")\n",
    "\n",
    "trainer.train()\n",
    "eval_results = trainer.evaluate()\n",
    "print(f\"Evaluation results for task {i+1}: {eval_results}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "iKp7_9I7bQv0"
   },
   "source": [
    "# Section 3.5 Exploring LoRA (Low-Rank Adaptation) for Task-Specific Fine-Tuning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 231,
   "metadata": {
    "id": "TBHbhNmpXyM8"
   },
   "outputs": [],
   "source": [
    "   import torch\n",
    "   import torch.nn as nn\n",
    "\n",
    "   class LoRALayer(nn.Module):\n",
    "       def __init__(self, input_dim, rank):\n",
    "           super(LoRALayer, self).__init__()\n",
    "           self.A = nn.Parameter(torch.randn(input_dim, rank))\n",
    "           self.B = nn.Parameter(torch.randn(rank, input_dim))\n",
    "\n",
    "       def forward(self, x):\n",
    "           return x + torch.matmul(torch.matmul(x, self.A), self.B)\n",
    "\n",
    "   class BertWithLoRA(nn.Module):\n",
    "       def __init__(self, model_name, rank):\n",
    "           super(BertWithLoRA, self).__init__()\n",
    "           self.bert = BertModel.from_pretrained(model_name)\n",
    "           self.lora_layers = nn.ModuleList([LoRALayer(self.bert.config.hidden_size, rank) for _ in range(self.bert.config.num_hidden_layers)])\n",
    "\n",
    "       def forward(self, input_ids, attention_mask=None):\n",
    "           outputs = self.bert(input_ids, attention_mask=attention_mask)\n",
    "           for i, lora_layer in enumerate(self.lora_layers):\n",
    "               outputs.last_hidden_state[:, i, :] = lora_layer(outputs.last_hidden_state[:, i, :])\n",
    "           return outputs\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 232,
   "metadata": {
    "id": "4B2cYBpwXL0p"
   },
   "outputs": [],
   "source": [
    "   from transformers import BertTokenizer, BertModel\n",
    "   from datasets import load_dataset\n",
    "\n",
    "   model_name = 'bert-base-uncased'\n",
    "   tokenizer = BertTokenizer.from_pretrained(model_name)\n",
    "   model = BertWithLoRA(model_name, rank=8)\n",
    "\n",
    "   dataset = load_dataset('glue', 'mrpc', split='train')\n",
    "\n",
    "   def preprocess(data):\n",
    "       return tokenizer(data['sentence1'], data['sentence2'], truncation=True, padding=True)\n",
    "\n",
    "   dataset = dataset.map(preprocess, batched=True)\n",
    "   dataset.set_format(type='torch', columns=['input_ids', 'attention_mask', 'label'])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 233,
   "metadata": {
    "id": "liykItFQ_tZK"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\r\n",
      "\u001b[0m"
     ]
    }
   ],
   "source": [
    "!pip install transformers datasets --quiet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 234,
   "metadata": {
    "id": "IRNqS69uXL33"
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f9fcf681f800415abec8eb5bc8c7461d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Map:   0%|          | 0/3668 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8aeae162af97401cad2bfc3d558fd700",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Map:   0%|          | 0/408 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.10/dist-packages/accelerate/accelerator.py:488: FutureWarning: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead.\n",
      "  self.scaler = torch.cuda.amp.GradScaler(**kwargs)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='690' max='690' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [690/690 00:50, Epoch 3/3]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Epoch</th>\n",
       "      <th>Training Loss</th>\n",
       "      <th>Validation Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.392980</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.365131</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>0.422300</td>\n",
       "      <td>0.447022</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='26' max='26' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [26/26 00:00]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluation results for task 2: {'eval_loss': 0.44702231884002686, 'eval_runtime': 0.2438, 'eval_samples_per_second': 1673.577, 'eval_steps_per_second': 106.65, 'epoch': 3.0}\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "from transformers import BertForSequenceClassification, BertModel, BertTokenizer\n",
    "from transformers import Trainer, TrainingArguments\n",
    "from datasets import load_dataset\n",
    "\n",
    "# Define your custom model\n",
    "class BertWithAdapters(torch.nn.Module):\n",
    "    def __init__(self, model_name, bottleneck_dim):\n",
    "        super().__init__()\n",
    "        self.bert = BertModel.from_pretrained(model_name)\n",
    "        self.tokenizer = BertTokenizer.from_pretrained(model_name) # Initialize the tokenizer here\n",
    "        self.bottleneck = torch.nn.Linear(self.bert.config.hidden_size, bottleneck_dim)\n",
    "        self.classifier = torch.nn.Linear(bottleneck_dim, 2) # Assuming binary classification\n",
    "\n",
    "    def forward(self, input_ids, attention_mask, labels=None): # Add labels as argument\n",
    "        outputs = self.bert(input_ids=input_ids, attention_mask=attention_mask)\n",
    "        pooled_output = outputs.pooler_output\n",
    "        bottleneck_output = self.bottleneck(pooled_output)\n",
    "        logits = self.classifier(bottleneck_output)\n",
    "\n",
    "        loss = None\n",
    "        if labels is not None: # Calculate loss if labels are provided\n",
    "            loss_fct = torch.nn.CrossEntropyLoss()\n",
    "            loss = loss_fct(logits.view(-1, 2), labels.view(-1)) # Assuming binary classification\n",
    "\n",
    "        return {'loss': loss, 'logits': logits} # Return loss and logits\n",
    "\n",
    "\n",
    "training_args = TrainingArguments(\n",
    "    output_dir='./results',\n",
    "    eval_strategy='epoch',\n",
    "    learning_rate=2e-5,\n",
    "    per_device_train_batch_size=16,\n",
    "    per_device_eval_batch_size=16,\n",
    "    num_train_epochs=3,\n",
    "    weight_decay=0.01,\n",
    "    logging_dir='./logs',\n",
    ")\n",
    "\n",
    "model_name = 'bert-base-uncased'\n",
    "bottleneck_dim = 64\n",
    "model = BertWithAdapters(model_name, bottleneck_dim)\n",
    "\n",
    "# Load both training and validation datasets\n",
    "dataset_train = load_dataset('glue', 'mrpc', split='train')\n",
    "dataset_val = load_dataset('glue', 'mrpc', split='validation') # Load the validation set\n",
    "\n",
    "# Tokenize the datasets\n",
    "def preprocess_function(examples):\n",
    "    return model.tokenizer(examples['sentence1'], examples['sentence2'], truncation=True, padding='longest') # Pad to the longest sequence in the batch\n",
    "\n",
    "dataset_train = dataset_train.map(preprocess_function, batched=True)\n",
    "dataset_val = dataset_val.map(preprocess_function, batched=True) # Tokenize the validation set\n",
    "\n",
    "# Add a data collator to prepare the inputs for the model\n",
    "from transformers import DataCollatorWithPadding\n",
    "data_collator = DataCollatorWithPadding(tokenizer=model.tokenizer) # Use the tokenizer associated with your BERT model\n",
    "\n",
    "trainer = Trainer(\n",
    "    model=model,\n",
    "    args=training_args,\n",
    "    train_dataset=dataset_train, # Pass the training dataset\n",
    "    eval_dataset=dataset_val,   # Pass the validation dataset\n",
    "    data_collator=data_collator # Pass the data collator to the Trainer\n",
    ")\n",
    "\n",
    "trainer.train()\n",
    "eval_results = trainer.evaluate()\n",
    "print(f\"Evaluation results for task {i+1}: {eval_results}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "7sKb0b-OtNZi"
   },
   "source": [
    "# Section 3.6 Finding the Balance Between Efficiency and Performance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 235,
   "metadata": {
    "id": "VIeY83W3XzCo"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of BertForSequenceClassification were not initialized from the model checkpoint at bert-base-uncased and are newly initialized: ['classifier.bias', 'classifier.weight']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    }
   ],
   "source": [
    "from transformers import BertForSequenceClassification, Trainer, TrainingArguments\n",
    "\n",
    "model = BertForSequenceClassification.from_pretrained('bert-base-uncased')\n",
    "# Example code for pruning would depend on specific pruning libraries/methods\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 236,
   "metadata": {
    "id": "Pe28Bc-dYvvY"
   },
   "outputs": [],
   "source": [
    "from transformers import DistilBertModel, DistilBertTokenizer\n",
    "\n",
    "tokenizer = DistilBertTokenizer.from_pretrained('distilbert-base-uncased')\n",
    "model = DistilBertModel.from_pretrained('distilbert-base-uncased')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 237,
   "metadata": {
    "id": "ptBb1VFKYy3f"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of BertForSequenceClassification were not initialized from the model checkpoint at bert-base-uncased and are newly initialized: ['classifier.bias', 'classifier.weight']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "BertForSequenceClassification(\n",
       "  (bert): BertModel(\n",
       "    (embeddings): BertEmbeddings(\n",
       "      (word_embeddings): Embedding(30522, 768, padding_idx=0)\n",
       "      (position_embeddings): Embedding(512, 768)\n",
       "      (token_type_embeddings): Embedding(2, 768)\n",
       "      (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "      (dropout): Dropout(p=0.1, inplace=False)\n",
       "    )\n",
       "    (encoder): BertEncoder(\n",
       "      (layer): ModuleList(\n",
       "        (0-11): 12 x BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSdpaSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (intermediate_act_fn): GELUActivation()\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "    )\n",
       "    (pooler): BertPooler(\n",
       "      (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "      (activation): Tanh()\n",
       "    )\n",
       "  )\n",
       "  (dropout): Dropout(p=0.1, inplace=False)\n",
       "  (classifier): Linear(in_features=768, out_features=2, bias=True)\n",
       ")"
      ]
     },
     "execution_count": 237,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from transformers import BertForSequenceClassification, Trainer, TrainingArguments\n",
    "\n",
    "model = BertForSequenceClassification.from_pretrained('bert-base-uncased')\n",
    "model.half()  # Convert model weights to 16-bit\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 238,
   "metadata": {
    "id": "rwRQzf3CYy88"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loss: 0.32245445251464844\n"
     ]
    }
   ],
   "source": [
    "from torch.amp import GradScaler, autocast\n",
    "from torch.utils.data import DataLoader, TensorDataset\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "\n",
    "# Sample training data (replace this with your actual data)\n",
    "train_data = [[1, 2, 3], [4, 5, 6]]\n",
    "train_labels = [0, 1]\n",
    "\n",
    "# Assuming 'train_data' and 'train_labels' are your training data and labels\n",
    "# Keep the input data as LongTensor (integers)\n",
    "train_dataset = TensorDataset(torch.tensor(train_data, dtype=torch.long), torch.tensor(train_labels))\n",
    "train_loader = DataLoader(train_dataset, batch_size=32, shuffle=True)\n",
    "\n",
    "# Define a simple model (replace this with your actual model)\n",
    "class SimpleModel(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(SimpleModel, self).__init__()\n",
    "        self.embedding = nn.Embedding(10, 8)\n",
    "        self.fc = nn.Linear(8 * 3, 2)  # Assuming input length of 3 and output classes of 2\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = self.embedding(x)\n",
    "        x = x.view(x.size(0), -1)\n",
    "        logits = self.fc(x)\n",
    "        return logits\n",
    "\n",
    "# Initialize the model\n",
    "model = SimpleModel().cuda()\n",
    "\n",
    "# Define loss function and optimizer\n",
    "criterion = nn.CrossEntropyLoss()\n",
    "optimizer = optim.Adam(model.parameters(), lr=1e-5)\n",
    "\n",
    "scaler = GradScaler('cuda')\n",
    "\n",
    "for data, target in train_loader:\n",
    "    optimizer.zero_grad()\n",
    "    data, target = data.cuda(), target.cuda()\n",
    "    with autocast('cuda'):\n",
    "        output = model(data)\n",
    "        # Ensure logits are in float32 before passing to criterion\n",
    "        loss = criterion(output.float(), target)\n",
    "    scaler.scale(loss).backward()\n",
    "    scaler.step(optimizer)\n",
    "    scaler.update()\n",
    "    print(f'Loss: {loss.item()}')\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Early Stopping"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 239,
   "metadata": {
    "id": "ZhjoDB98NgZ2"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.10/dist-packages/accelerate/accelerator.py:488: FutureWarning: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead.\n",
      "  self.scaler = torch.cuda.amp.GradScaler(**kwargs)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='4' max='10' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [ 4/10 00:00 < 00:01, 3.91 it/s, Epoch 4/10]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Epoch</th>\n",
       "      <th>Training Loss</th>\n",
       "      <th>Validation Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>No log</td>\n",
       "      <td>1.183105</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>No log</td>\n",
       "      <td>1.183105</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>No log</td>\n",
       "      <td>1.183350</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>No log</td>\n",
       "      <td>1.183350</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='1' max='1' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [1/1 : < :]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluation results for task 2: {'eval_loss': 1.18310546875, 'eval_runtime': 0.0039, 'eval_samples_per_second': 506.65, 'eval_steps_per_second': 253.325, 'epoch': 4.0}\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "from torch.cuda.amp import GradScaler, autocast\n",
    "from torch.utils.data import DataLoader, TensorDataset\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "from transformers import EarlyStoppingCallback, TrainingArguments, Trainer, default_data_collator\n",
    "\n",
    "os.environ['CUDA_LAUNCH_BLOCKING'] = '1'\n",
    "\n",
    "# Sample training data (replace this with your actual data)\n",
    "train_data = [[1, 2, 3], [4, 5, 6]]\n",
    "train_labels = [0, 1]\n",
    "\n",
    "# Sample evaluation data (replace this with your actual data)\n",
    "eval_data = [[7, 8, 9], [10, 11, 12]]\n",
    "eval_labels = [1, 0]\n",
    "\n",
    "# Convert the data to a list of dictionaries\n",
    "# Each dictionary represents a single data point with 'input_ids' and 'labels' keys\n",
    "train_dataset_for_trainer = [\n",
    "    {\"input_ids\": torch.tensor(data, dtype=torch.long),\n",
    "     \"labels\": torch.tensor(label, dtype=torch.long)}  # Make sure labels are also tensors of dtype long\n",
    "    for data, label in zip(train_data, train_labels)\n",
    "]\n",
    "\n",
    "eval_dataset_for_trainer = [\n",
    "    {\"input_ids\": torch.tensor(data, dtype=torch.long),\n",
    "     \"labels\": torch.tensor(label, dtype=torch.long)}\n",
    "    for data, label in zip(eval_data, eval_labels)\n",
    "]\n",
    "\n",
    "# Define a simple model (replace this with your actual model)\n",
    "class SimpleModel(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(SimpleModel, self).__init__()\n",
    "        self.embedding = nn.Embedding(13, 8)  # Adjust embedding dimension for the vocabulary size\n",
    "        self.fc = nn.Linear(8 * 3, 2)  # Assuming input length of 3 and output classes of 2\n",
    "\n",
    "    def forward(self, input_ids=None, labels=None):\n",
    "        # Debugging: Check input_ids and print if assertion fails\n",
    "        if input_ids is not None:\n",
    "            if not ((input_ids >= 0).all() and (input_ids < 13).all()):\n",
    "                print(\"Warning: input_ids out of bounds:\", input_ids)\n",
    "\n",
    "        x = self.embedding(input_ids)\n",
    "        x = x.view(x.size(0), -1)\n",
    "        logits = self.fc(x)\n",
    "        loss = None\n",
    "        if labels is not None:\n",
    "            loss_fct = nn.CrossEntropyLoss()\n",
    "            # No need to reshape logits, CrossEntropyLoss expects raw logits and class indices\n",
    "            loss = loss_fct(logits, labels)\n",
    "        return {'loss': loss, 'logits': logits}\n",
    "\n",
    "# Initialize the model\n",
    "model = SimpleModel()\n",
    "model = model.cuda()  # Comment this line to run on CPU\n",
    "\n",
    "# Training arguments\n",
    "training_args = TrainingArguments(\n",
    "    output_dir='./results',\n",
    "    eval_strategy='epoch',  # Use the updated argument name\n",
    "    learning_rate=2e-5,\n",
    "    per_device_train_batch_size=16,\n",
    "    per_device_eval_batch_size=16,\n",
    "    num_train_epochs=10,\n",
    "    weight_decay=0.01,\n",
    "    logging_dir='./logs',\n",
    "    load_best_model_at_end=True,\n",
    "    save_strategy='epoch'\n",
    ")\n",
    "\n",
    "# Create the Trainer object\n",
    "trainer = Trainer(\n",
    "    model=model,\n",
    "    args=training_args,\n",
    "    train_dataset=train_dataset_for_trainer,  # Use the correct dataset variable\n",
    "    eval_dataset=eval_dataset_for_trainer,\n",
    "    callbacks=[EarlyStoppingCallback(early_stopping_patience=3)],\n",
    "    data_collator=default_data_collator\n",
    ")\n",
    "\n",
    "trainer.train()\n",
    "eval_results = trainer.evaluate()\n",
    "print(f\"Evaluation results for task {i+1}: {eval_results}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 240,
   "metadata": {
    "id": "HzzgrkxyY9oX"
   },
   "outputs": [],
   "source": [
    "from torch.utils.data import DataLoader\n",
    "from datasets import load_dataset\n",
    "from transformers import EarlyStoppingCallback, TrainingArguments, Trainer, default_data_collator, DistilBertTokenizer # Import the tokenizer\n",
    "\n",
    "tokenizer = DistilBertTokenizer.from_pretrained('distilbert-base-uncased')\n",
    "dataset = load_dataset('glue', 'mrpc', split='train')\n",
    "\n",
    "def preprocess(data):\n",
    "  return tokenizer(data['sentence1'], data['sentence2'], truncation=True, padding=True)\n",
    "\n",
    "dataset = dataset.map(preprocess, batched=True)\n",
    "\n",
    "train_loader = DataLoader(dataset, batch_size=32, shuffle=True, num_workers=4, pin_memory=True)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "MMTeLkAnZC4u"
   },
   "source": [
    "Practical Example"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "vAnC3LUyZWav"
   },
   "source": [
    "1. Model Pruning and Knowledge Distillation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 241,
   "metadata": {
    "id": "bj1EJ8fMZBsY"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of BertForSequenceClassification were not initialized from the model checkpoint at bert-base-uncased and are newly initialized: ['classifier.bias', 'classifier.weight']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
      "Some weights of DistilBertForSequenceClassification were not initialized from the model checkpoint at distilbert-base-uncased and are newly initialized: ['classifier.bias', 'classifier.weight', 'pre_classifier.bias', 'pre_classifier.weight']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
      "/usr/local/lib/python3.10/dist-packages/accelerate/accelerator.py:488: FutureWarning: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead.\n",
      "  self.scaler = torch.cuda.amp.GradScaler(**kwargs)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='690' max='690' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [690/690 10:01, Epoch 3/3]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Epoch</th>\n",
       "      <th>Training Loss</th>\n",
       "      <th>Validation Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>0.559400</td>\n",
       "      <td>0.524520</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>0.486600</td>\n",
       "      <td>0.511143</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>0.460700</td>\n",
       "      <td>0.516225</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='26' max='26' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [26/26 00:00]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluation results for task 2: {'eval_loss': 0.5162246227264404, 'eval_runtime': 0.6101, 'eval_samples_per_second': 668.762, 'eval_steps_per_second': 42.617, 'epoch': 3.0}\n"
     ]
    }
   ],
   "source": [
    "from transformers import BertForSequenceClassification, DistilBertForSequenceClassification, Trainer, TrainingArguments\n",
    "from datasets import load_dataset\n",
    "from transformers import EarlyStoppingCallback, default_data_collator, DistilBertTokenizer\n",
    "import torch\n",
    "from torch.nn import functional as F\n",
    "from torch.utils.data import DataLoader\n",
    "\n",
    "# Define device\n",
    "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
    "\n",
    "# Load datasets\n",
    "dataset_train = load_dataset('glue', 'mrpc', split='train')\n",
    "dataset_val = load_dataset('glue', 'mrpc', split='validation')\n",
    "\n",
    "# Load tokenizer\n",
    "tokenizer = DistilBertTokenizer.from_pretrained('distilbert-base-uncased')\n",
    "\n",
    "# Tokenize the datasets\n",
    "def preprocess(data):\n",
    "    return tokenizer(data['sentence1'], data['sentence2'], truncation=True, padding='max_length', max_length=100)\n",
    "\n",
    "dataset_train = dataset_train.map(preprocess, batched=True)\n",
    "dataset_val = dataset_val.map(preprocess, batched=True)\n",
    "\n",
    "dataset_train.set_format(type='torch', columns=['input_ids', 'attention_mask', 'label'])\n",
    "dataset_val.set_format(type='torch', columns=['input_ids', 'attention_mask', 'label'])\n",
    "\n",
    "# Load teacher model\n",
    "teacher_model = BertForSequenceClassification.from_pretrained('bert-base-uncased').to(device)\n",
    "\n",
    "# Load student model\n",
    "student_model = DistilBertForSequenceClassification.from_pretrained('distilbert-base-uncased').to(device)\n",
    "\n",
    "\n",
    "# Define training arguments\n",
    "training_args = TrainingArguments(\n",
    "    output_dir='./results',\n",
    "    eval_strategy='epoch',\n",
    "    learning_rate=2e-5,\n",
    "    per_device_train_batch_size=16,\n",
    "    per_device_eval_batch_size=16,\n",
    "    num_train_epochs=3,\n",
    "    weight_decay=0.01,\n",
    "    logging_dir='./logs',\n",
    "    logging_steps=10,\n",
    "    save_steps=10,\n",
    ")\n",
    "\n",
    "# Distillation loss\n",
    "def distillation_loss(student_outputs, teacher_outputs, temperature=2.0):\n",
    "    soft_labels = nn.functional.softmax(teacher_outputs.logits / temperature, dim=-1)\n",
    "    student_logits = student_outputs.logits / temperature\n",
    "    return nn.functional.kl_div(nn.functional.log_softmax(student_logits, dim=-1), soft_labels, reduction='batchmean') * (temperature ** 2)\n",
    "\n",
    "# Define custom Trainer for distillation\n",
    "class DistillationTrainer(Trainer):\n",
    "    def compute_loss(self, model, inputs, return_outputs=False):\n",
    "        labels = inputs.pop(\"labels\")\n",
    "        student_outputs = model(**inputs)\n",
    "        with torch.no_grad():\n",
    "            teacher_outputs = teacher_model(**inputs)\n",
    "        loss = distillation_loss(student_outputs, teacher_outputs) + nn.functional.cross_entropy(student_outputs.logits, labels)\n",
    "        return (loss, student_outputs) if return_outputs else loss\n",
    "\n",
    "\n",
    "# Define the Trainer\n",
    "trainer = DistillationTrainer(\n",
    "    model=student_model,\n",
    "    args=training_args,\n",
    "    train_dataset=dataset_train,\n",
    "    eval_dataset=dataset_val,\n",
    "    data_collator=default_data_collator,\n",
    ")\n",
    "\n",
    "# Train the student model\n",
    "trainer.train()\n",
    "eval_results = trainer.evaluate()\n",
    "print(f\"Evaluation results for task {i+1}: {eval_results}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Prune the student model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 243,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Parameter containing:\n",
      "tensor([[-0.0000, -0.0000, -0.0196,  ...,  0.0000,  0.0250, -0.0458],\n",
      "        [-0.0227, -0.0474,  0.0000,  ...,  0.0000, -0.0163, -0.0000]],\n",
      "       device='cuda:0', requires_grad=True)\n"
     ]
    }
   ],
   "source": [
    "import torch.nn.utils.prune as prune\n",
    "from transformers import DistilBertForSequenceClassification\n",
    "\n",
    "# Example of pruning a linear layer in a neural network\n",
    "# DistilBertForSequenceClassification typically has a single classifier layer named 'classifier'\n",
    "module = student_model.classifier  # Access the classifier layer directly\n",
    "\n",
    "# Prune 40% of the weights in the classifier layer\n",
    "prune.l1_unstructured(module, name='weight', amount=0.4)\n",
    "\n",
    "# Optional: Remove the pruning reparameterization to make pruning permanent\n",
    "prune.remove(module, 'weight')\n",
    "\n",
    "# Verify the pruning\n",
    "print(module.weight)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "hSVPJdExZUcY"
   },
   "source": [
    "2. Mixed Precision Training and Early Stopping"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 244,
   "metadata": {
    "id": "ReX5klshZG4f"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of BertForSequenceClassification were not initialized from the model checkpoint at bert-base-uncased and are newly initialized: ['classifier.bias', 'classifier.weight']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
      "/usr/local/lib/python3.10/dist-packages/accelerate/accelerator.py:488: FutureWarning: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead.\n",
      "  self.scaler = torch.cuda.amp.GradScaler(**kwargs)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='2000' max='2300' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [2000/2300 03:03 < 00:27, 10.86 it/s, Epoch 8/10]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "      <th>Validation Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>500</td>\n",
       "      <td>0.448400</td>\n",
       "      <td>0.408246</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1000</td>\n",
       "      <td>0.169100</td>\n",
       "      <td>0.707166</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1500</td>\n",
       "      <td>0.058300</td>\n",
       "      <td>0.984032</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2000</td>\n",
       "      <td>0.019800</td>\n",
       "      <td>1.070677</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='26' max='26' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [26/26 00:00]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluation results for task 2: {'eval_loss': 0.4082459807395935, 'eval_runtime': 0.4089, 'eval_samples_per_second': 997.843, 'eval_steps_per_second': 63.588, 'epoch': 8.695652173913043}\n"
     ]
    }
   ],
   "source": [
    "from transformers import BertForSequenceClassification, Trainer, TrainingArguments, default_data_collator\n",
    "from torch.cuda.amp import GradScaler, autocast\n",
    "\n",
    "model = BertForSequenceClassification.from_pretrained('bert-base-uncased')\n",
    "\n",
    "def preprocess(data):\n",
    "  return tokenizer(data['sentence1'], data['sentence2'], truncation=True, padding='max_length', max_length=100)\n",
    "\n",
    "dataset_train = dataset_train.map(preprocess, batched=True)\n",
    "dataset_val = dataset_val.map(preprocess, batched=True)\n",
    "\n",
    "# Define training arguments\n",
    "training_args = TrainingArguments(\n",
    "    output_dir='./results',\n",
    "    eval_strategy='steps',\n",
    "    learning_rate=2e-5,\n",
    "    per_device_train_batch_size=16,\n",
    "    per_device_eval_batch_size=16,\n",
    "    num_train_epochs=10,\n",
    "    weight_decay=0.01,\n",
    "    logging_dir='./logs',\n",
    "    load_best_model_at_end=True,\n",
    "    fp16=True  # Enable mixed-precision training\n",
    ")\n",
    "\n",
    "trainer = Trainer(\n",
    "    model=model,\n",
    "    args=training_args,\n",
    "    train_dataset=dataset_train,\n",
    "    eval_dataset=dataset_val,\n",
    "    # Use the default data collator to handle variable sequence lengths\n",
    "    data_collator=default_data_collator,\n",
    "    callbacks=[EarlyStoppingCallback(early_stopping_patience=3)]\n",
    ")\n",
    "\n",
    "trainer.train()\n",
    "eval_results = trainer.evaluate()\n",
    "print(f\"Evaluation results for task {i+1}: {eval_results}\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "80uj-ulPTF5T"
   },
   "source": [
    "# Section 3.7 Moving beyond PEFT and LoRA"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "SkKXvwSDZuZn"
   },
   "source": [
    "1. Few-shot Learning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 245,
   "metadata": {
    "id": "IdPjgUvDXzrM"
   },
   "outputs": [
    {
     "ename": "AuthenticationError",
     "evalue": "Error code: 401 - {'error': {'message': 'Incorrect API key provided: <YOUR KE***ERE>. You can find your API key at https://platform.openai.com/account/api-keys.', 'type': 'invalid_request_error', 'param': None, 'code': 'invalid_api_key'}}",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mAuthenticationError\u001b[0m                       Traceback (most recent call last)",
      "\u001b[0;32m/tmp/ipykernel_224772/561274127.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     25\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     26\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 27\u001b[0;31m response = client.chat.completions.create(\n\u001b[0m\u001b[1;32m     28\u001b[0m         \u001b[0mmodel\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"gpt-4o\"\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     29\u001b[0m         messages=[\n",
      "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/openai/_utils/_utils.py\u001b[0m in \u001b[0;36mwrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m    275\u001b[0m                         \u001b[0mmsg\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34mf\"Missing required argument: {quote(missing[0])}\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    276\u001b[0m                 \u001b[0;32mraise\u001b[0m \u001b[0mTypeError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmsg\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 277\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    278\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    279\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mwrapper\u001b[0m  \u001b[0;31m# type: ignore\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/openai/resources/chat/completions.py\u001b[0m in \u001b[0;36mcreate\u001b[0;34m(self, messages, model, frequency_penalty, function_call, functions, logit_bias, logprobs, max_tokens, n, parallel_tool_calls, presence_penalty, response_format, seed, service_tier, stop, stream, stream_options, temperature, tool_choice, tools, top_logprobs, top_p, user, extra_headers, extra_query, extra_body, timeout)\u001b[0m\n\u001b[1;32m    644\u001b[0m         \u001b[0mtimeout\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mfloat\u001b[0m \u001b[0;34m|\u001b[0m \u001b[0mhttpx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTimeout\u001b[0m \u001b[0;34m|\u001b[0m \u001b[0;32mNone\u001b[0m \u001b[0;34m|\u001b[0m \u001b[0mNotGiven\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mNOT_GIVEN\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    645\u001b[0m     ) -> ChatCompletion | Stream[ChatCompletionChunk]:\n\u001b[0;32m--> 646\u001b[0;31m         return self._post(\n\u001b[0m\u001b[1;32m    647\u001b[0m             \u001b[0;34m\"/chat/completions\"\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    648\u001b[0m             body=maybe_transform(\n",
      "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/openai/_base_client.py\u001b[0m in \u001b[0;36mpost\u001b[0;34m(self, path, cast_to, body, options, files, stream, stream_cls)\u001b[0m\n\u001b[1;32m   1264\u001b[0m             \u001b[0mmethod\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"post\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0murl\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mjson_data\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mbody\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfiles\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mto_httpx_files\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfiles\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0moptions\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1265\u001b[0m         )\n\u001b[0;32m-> 1266\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mcast\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mResponseT\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrequest\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcast_to\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mopts\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstream\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mstream\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstream_cls\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mstream_cls\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1267\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1268\u001b[0m     def patch(\n",
      "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/openai/_base_client.py\u001b[0m in \u001b[0;36mrequest\u001b[0;34m(self, cast_to, options, remaining_retries, stream, stream_cls)\u001b[0m\n\u001b[1;32m    940\u001b[0m         \u001b[0mstream_cls\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mtype\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0m_StreamT\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m|\u001b[0m \u001b[0;32mNone\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    941\u001b[0m     ) -> ResponseT | _StreamT:\n\u001b[0;32m--> 942\u001b[0;31m         return self._request(\n\u001b[0m\u001b[1;32m    943\u001b[0m             \u001b[0mcast_to\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcast_to\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    944\u001b[0m             \u001b[0moptions\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0moptions\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/openai/_base_client.py\u001b[0m in \u001b[0;36m_request\u001b[0;34m(self, cast_to, options, remaining_retries, stream, stream_cls)\u001b[0m\n\u001b[1;32m   1044\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1045\u001b[0m             \u001b[0mlog\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdebug\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Re-raising status error\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1046\u001b[0;31m             \u001b[0;32mraise\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_make_status_error_from_response\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0merr\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mresponse\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1047\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1048\u001b[0m         return self._process_response(\n",
      "\u001b[0;31mAuthenticationError\u001b[0m: Error code: 401 - {'error': {'message': 'Incorrect API key provided: <YOUR KE***ERE>. You can find your API key at https://platform.openai.com/account/api-keys.', 'type': 'invalid_request_error', 'param': None, 'code': 'invalid_api_key'}}"
     ]
    }
   ],
   "source": [
    "from openai import OpenAI\n",
    "#from google.colab import userdata\n",
    "\n",
    "#client = OpenAI(api_key=userdata.get('open_ai_key'))\n",
    "client = OpenAI(api_key=\"<YOUR KEY HERE>\")\n",
    "\n",
    "# Define a few-shot learning prompt\n",
    "prompt = \"\"\"\n",
    "The following is a conversation with an AI assistant. The assistant is helpful, creative, clever, and very friendly.\n",
    "\n",
    "Human: Hello, who are you?\n",
    "AI: I am an AI created by OpenAI. How can I help you today?\n",
    "Human: Can you tell me a joke?\n",
    "AI: Sure! Why don't scientists trust atoms? Because they make up everything!\n",
    "\n",
    "Human: What is the capital of France?\n",
    "AI: The capital of France is Paris.\n",
    "\n",
    "Human: How does photosynthesis work?\n",
    "AI: Photosynthesis is the process by which green plants and some other organisms use sunlight to synthesize foods with the help of chlorophyll from carbon dioxide and water.\n",
    "\"\"\"\n",
    "\n",
    "# Add your query to the prompt\n",
    "prompt += \"\\nHuman: What is the largest planet in our solar system?\\nAI:\"\n",
    "\n",
    "\n",
    "response = client.chat.completions.create(\n",
    "        model=\"gpt-4o\",\n",
    "        messages=[\n",
    "            {\"role\": \"system\", \"content\": \"You are a helpful assistant that answers questions.\"},\n",
    "            {\"role\": \"user\", \"content\": str(prompt) }\n",
    "        ],\n",
    "        max_tokens=150,\n",
    "        temperature=0.7,\n",
    "    )\n",
    "\n",
    "print(response.choices[0].message.content.strip())\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "6RqVX2D8Z4Pg"
   },
   "source": [
    "2. Transfer Learning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 246,
   "metadata": {
    "id": "ClEmiLCpZtQi"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of BertForSequenceClassification were not initialized from the model checkpoint at bert-base-uncased and are newly initialized: ['classifier.bias', 'classifier.weight']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
      "/usr/local/lib/python3.10/dist-packages/accelerate/accelerator.py:488: FutureWarning: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead.\n",
      "  self.scaler = torch.cuda.amp.GradScaler(**kwargs)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='12630' max='12630' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [12630/12630 17:24, Epoch 3/3]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Epoch</th>\n",
       "      <th>Training Loss</th>\n",
       "      <th>Validation Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>0.176800</td>\n",
       "      <td>0.249657</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>0.123800</td>\n",
       "      <td>0.302528</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>0.070400</td>\n",
       "      <td>0.324817</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='55' max='55' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [55/55 00:00]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluation results for task 2: {'eval_loss': 0.3248170018196106, 'eval_runtime': 0.4288, 'eval_samples_per_second': 2033.37, 'eval_steps_per_second': 128.252, 'epoch': 3.0}\n"
     ]
    }
   ],
   "source": [
    "from transformers import BertForSequenceClassification, BertTokenizer\n",
    "from datasets import load_dataset\n",
    "\n",
    "model_name = 'bert-base-uncased'\n",
    "tokenizer = BertTokenizer.from_pretrained(model_name)\n",
    "model = BertForSequenceClassification.from_pretrained(model_name)\n",
    "\n",
    "# Load and preprocess dataset\n",
    "dataset = load_dataset('glue', 'sst2', split='train')\n",
    "dataset_validation = load_dataset('glue', 'sst2', split='validation')\n",
    "\n",
    "def preprocess(data):\n",
    "    return tokenizer(data['sentence'], truncation=True, padding='max_length', max_length=61, return_tensors='pt')\n",
    "\n",
    "dataset = dataset.map(preprocess, batched=True)\n",
    "dataset.set_format(type='torch', columns=['input_ids', 'attention_mask', 'label'])\n",
    "dataset_validation = dataset_validation.map(preprocess, batched=True)\n",
    "dataset_validation.set_format(type='torch', columns=['input_ids', 'attention_mask', 'label'])\n",
    "\n",
    "# Fine-tune on the new task\n",
    "from transformers import Trainer, TrainingArguments\n",
    "training_args = TrainingArguments(\n",
    "    output_dir='./results',\n",
    "    eval_strategy='epoch',\n",
    "    learning_rate=2e-5,\n",
    "    per_device_train_batch_size=16,\n",
    "    per_device_eval_batch_size=16,\n",
    "    num_train_epochs=3,\n",
    "    weight_decay=0.01,\n",
    "    logging_dir='./logs',\n",
    ")\n",
    "\n",
    "trainer = Trainer(\n",
    "    model=model,\n",
    "    args=training_args,\n",
    "    train_dataset=dataset,\n",
    "    eval_dataset=dataset_validation,\n",
    ")\n",
    "\n",
    "trainer.train()\n",
    "\n",
    "eval_results = trainer.evaluate()\n",
    "print(f\"Evaluation results for task {i+1}: {eval_results}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "kyhKfatQZ-J_"
   },
   "source": [
    "3. Continual Learning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 247,
   "metadata": {
    "id": "iWzQ9tndZtWM"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of BertForSequenceClassification were not initialized from the model checkpoint at bert-base-uncased and are newly initialized: ['classifier.bias', 'classifier.weight']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
      "/usr/local/lib/python3.10/dist-packages/accelerate/accelerator.py:488: FutureWarning: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead.\n",
      "  self.scaler = torch.cuda.amp.GradScaler(**kwargs)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='12630' max='12630' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [12630/12630 17:32, Epoch 3/3]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Epoch</th>\n",
       "      <th>Training Loss</th>\n",
       "      <th>Validation Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>0.178200</td>\n",
       "      <td>0.243762</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>0.128000</td>\n",
       "      <td>0.297644</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>0.073100</td>\n",
       "      <td>0.327843</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='55' max='55' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [55/55 00:00]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluation results for task 1: {'eval_loss': 0.3278431296348572, 'eval_runtime': 0.4495, 'eval_samples_per_second': 1939.952, 'eval_steps_per_second': 122.359, 'epoch': 3.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.10/dist-packages/accelerate/accelerator.py:488: FutureWarning: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead.\n",
      "  self.scaler = torch.cuda.amp.GradScaler(**kwargs)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='690' max='690' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [690/690 00:52, Epoch 3/3]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Epoch</th>\n",
       "      <th>Training Loss</th>\n",
       "      <th>Validation Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.452498</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.423250</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>0.479900</td>\n",
       "      <td>0.491157</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='26' max='26' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [26/26 00:00]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluation results for task 2: {'eval_loss': 0.49115684628486633, 'eval_runtime': 0.2372, 'eval_samples_per_second': 1720.117, 'eval_steps_per_second': 109.615, 'epoch': 3.0}\n"
     ]
    }
   ],
   "source": [
    "from transformers import BertForSequenceClassification, BertTokenizer, Trainer, TrainingArguments\n",
    "from datasets import load_dataset\n",
    "\n",
    "model_name = 'bert-base-uncased'\n",
    "tokenizer = BertTokenizer.from_pretrained(model_name)\n",
    "model = BertForSequenceClassification.from_pretrained(model_name)\n",
    "\n",
    "# Simulate continual learning with multiple tasks\n",
    "datasets = [load_dataset('glue', 'sst2', split='train'), load_dataset('glue', 'mrpc', split='train')]\n",
    "datasets_validation = [load_dataset('glue', 'sst2', split='validation'), load_dataset('glue', 'mrpc', split='validation')]\n",
    "\n",
    "def preprocess(data):\n",
    "    if 'sentence' in data:\n",
    "        return tokenizer(data['sentence'], truncation=True, return_overflowing_tokens=False, padding='max_length', max_length=61, return_tensors='pt')\n",
    "    else:\n",
    "        return tokenizer(data['sentence1'], data['sentence2'], truncation=True, return_overflowing_tokens=False, padding='max_length', max_length=61, return_tensors='pt')\n",
    "\n",
    "for i in range(0,len(datasets)):\n",
    "    dataset = datasets[i].map(preprocess, batched=True)\n",
    "    dataset.set_format(type='torch', columns=['input_ids', 'attention_mask', 'label'])\n",
    "    dataset_validation = datasets_validation[i].map(preprocess, batched=True)\n",
    "    dataset_validation.set_format(type='torch', columns=['input_ids', 'attention_mask', 'label'])\n",
    "\n",
    "    trainer = Trainer(\n",
    "        model=model,\n",
    "        args=training_args,\n",
    "        train_dataset=dataset,\n",
    "        eval_dataset=dataset_validation\n",
    "    )\n",
    "\n",
    "    trainer.train()\n",
    "\n",
    "    eval_results = trainer.evaluate()\n",
    "    print(f\"Evaluation results for task {i+1}: {eval_results}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "qwquGVa-aHAJ"
   },
   "source": [
    "4. Meta-learning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 248,
   "metadata": {
    "id": "IsjNvCQgaeCA"
   },
   "outputs": [],
   "source": [
    "# Meta-learning frameworks like MAML (Model-Agnostic Meta-Learning) require specialized libraries and more complex setups."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "Uiwoa2NYafsG"
   },
   "source": [
    "Future Trends and Research Directions"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "LPcwpZ-ea2BR"
   },
   "source": [
    "2. Federated Learning:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 251,
   "metadata": {
    "id": "iR2RFCvra63_"
   },
   "outputs": [],
   "source": [
    "      # Federated learning involves complex setups and frameworks like PySyft for implementation."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "hmlHTPwGbCnQ"
   },
   "source": [
    "3. Energy-Efficient Models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 253,
   "metadata": {
    "id": "9_cPChz9bCV4"
   },
   "outputs": [],
   "source": [
    "# Techniques involve model pruning, quantization, and efficient data loading, as discussed in previous sections."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "uDKX6bOMbNC5"
   },
   "source": [
    "Practical Examples"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "cRHNPJkubVT_"
   },
   "source": [
    "1. Transfer Learning with BERT:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 255,
   "metadata": {
    "id": "1bNYko6ubUlP"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of BertForSequenceClassification were not initialized from the model checkpoint at bert-base-uncased and are newly initialized: ['classifier.bias', 'classifier.weight']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5a7edc29b0b04c34bf6f680f422ed953",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Map:   0%|          | 0/67349 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1b9c35532ce643dab8728d4bfb3ea3b0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Map:   0%|          | 0/872 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.10/dist-packages/accelerate/accelerator.py:488: FutureWarning: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead.\n",
      "  self.scaler = torch.cuda.amp.GradScaler(**kwargs)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='12630' max='12630' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [12630/12630 05:05, Epoch 3/3]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Epoch</th>\n",
       "      <th>Training Loss</th>\n",
       "      <th>Validation Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>0.655400</td>\n",
       "      <td>0.659729</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>0.641100</td>\n",
       "      <td>0.638516</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>0.638400</td>\n",
       "      <td>0.635563</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='55' max='55' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [55/55 00:00]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluation results: {'eval_loss': 0.6355632543563843, 'eval_runtime': 0.5488, 'eval_samples_per_second': 1588.923, 'eval_steps_per_second': 100.219, 'epoch': 3.0}\n"
     ]
    }
   ],
   "source": [
    "from transformers import BertForSequenceClassification, BertTokenizer, Trainer, TrainingArguments\n",
    "from datasets import load_dataset\n",
    "\n",
    "# Load pre-trained model and tokenizer\n",
    "model_name = 'bert-base-uncased'\n",
    "tokenizer = BertTokenizer.from_pretrained(model_name)\n",
    "model = BertForSequenceClassification.from_pretrained(model_name)\n",
    "\n",
    "# Freeze all layers except for the classification layer\n",
    "for param in model.bert.parameters():\n",
    "    param.requires_grad = False\n",
    "\n",
    "# Load datasets\n",
    "dataset = load_dataset('glue', 'sst2', split='train')\n",
    "dataset_validation = load_dataset('glue', 'sst2', split='validation')\n",
    "\n",
    "# Preprocess the data\n",
    "def preprocess(data):\n",
    "    return tokenizer(data['sentence'], truncation=True, padding='max_length', max_length=128)\n",
    "\n",
    "dataset = dataset.map(preprocess, batched=True)\n",
    "dataset.set_format(type='torch', columns=['input_ids', 'attention_mask', 'label'])\n",
    "dataset_validation = dataset_validation.map(preprocess, batched=True)\n",
    "dataset_validation.set_format(type='torch', columns=['input_ids', 'attention_mask', 'label'])\n",
    "\n",
    "# Define training arguments\n",
    "training_args = TrainingArguments(\n",
    "    output_dir='./results',\n",
    "    eval_strategy='epoch',\n",
    "    learning_rate=2e-5,\n",
    "    per_device_train_batch_size=16,\n",
    "    per_device_eval_batch_size=16,\n",
    "    num_train_epochs=3,\n",
    "    weight_decay=0.01,\n",
    "    logging_dir='./logs',\n",
    ")\n",
    "\n",
    "optimizer = AdamW([\n",
    "    {'params': model.bert.parameters(), 'lr': 1e-5},  # Lower learning rate for frozen layers\n",
    "    {'params': model.classifier.parameters(), 'lr': 2e-5}  # Higher learning rate for the classifier\n",
    "])\n",
    "\n",
    "\n",
    "# Define the Trainer\n",
    "trainer = Trainer(\n",
    "    model=model,\n",
    "    args=training_args,\n",
    "    train_dataset=dataset,\n",
    "    eval_dataset=dataset_validation,\n",
    "    optimizers=(optimizer, None)  # Pass optimizer (scheduler can be None or a custom scheduler)\n",
    ")\n",
    "\n",
    "# Train the model\n",
    "trainer.train()\n",
    "\n",
    "# Evaluate the model\n",
    "eval_results = trainer.evaluate()\n",
    "print(f\"Evaluation results: {eval_results}\")\n"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [
    "BcLSRzzIgepr",
    "o3mYU5CfSB-Y"
   ],
   "machine_shape": "hm",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
